Merge remote-tracking branch 'upstream/master'

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+ignore = F401, C901, F901
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist
+max-complexity = 10

.github/workflows/test.yml

@@ -0,0 +1,36 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [2.7, 3.6, 3.7, 3.8]
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest check-manifest sphinx
+    - name: Install the package
+      run: pip install -q -e .[dev]
+    - name: Run tests
+      run: |
+        pytest . -rs -q
+        check-manifest -v
+        sphinx-build docs _build -q -E
+        flake8 dbfread

CONTRIBUTING.md

@@ -0,0 +1,13 @@
+# Contributing to dbfread
+
+There will be more here later. For now I just wanted to add something about
+the local tox environments.
+
+## Running Test Locally
+
+There's a tox setup for running tests locally. If you don't have all
+the installed Python version you can run selected ones with:
+
+```
+tox -e py27,py38
+```

MANIFEST.in

@@ -1,7 +1,8 @@
-include README.rst LICENSE
+include README.rst LICENSE CONTRIBUTING.md tox.ini .flake8
 include examples/dbf2sqlite
 recursive-include examples *.dbf *.py
-recursive-include testcases *.dbf *.FPT
+recursive-include tests *.py
+recursive-include tests/cases *.dbf *.FPT
 recursive-include docs *.bat
 recursive-include docs *.py
 recursive-include docs *.rst

README.rst

@@ -15,17 +15,29 @@ FoxBase+. This library reads DBF files and returns the data as native
 Python data types for further processing. It is primarily intended for
 batch jobs and one-off scripts.
 
-::
+.. code-block:: python
 
     >>> from dbfread import DBF
+    >>> for record in DBF('people.dbf'):
+    ...     print(record)
+    {'NAME': 'Alice', 'BIRTHDATE': datetime.date(1987, 3, 1)}
+    {'NAME': 'Bob', 'BIRTHDATE': datetime.date(1980, 11, 12)}
+
+In older versions where dictionaries are not ordered you will instead get a
+``collections.OrderedDict``:
+
+.. code-block:: python
+
     >>> for record in DBF('people.dbf'):
     ...     print(record)
     OrderedDict([('NAME', 'Alice'), ('BIRTHDATE', datetime.date(1987, 3, 1))])
     OrderedDict([('NAME', 'Bob'), ('BIRTHDATE', datetime.date(1980, 11, 12))])
 
 By default records are streamed directly from the file.  If you have
 enough memory you can instead load them into a list. This allows for
-random access::
+random access:
+
+.. code-block:: python
 
     >>> table = DBF('people.dbf', load=True)
     >>> print(table.records[1]['NAME'])

docs/changes.rst

@@ -4,6 +4,25 @@ Changes
 Release History
 ---------------
 
+(Next Version)
+^^^^^^^^^^^^^^^^^^
+
+* records are now returned ``dict`` instead of ``collections.OrderedDict``
+  in Python 3.7 and up (as well as CPython 3.6) since normal Python
+  dictionaries are now ordered.
+
+* null values in D, F and L fields that are padded with ``'\0'`` bytes
+  should now work return ``None``. (Fix by william-andre, pull request #35.)
+
+* dbf2sqlite: table name was not escaped for DROP TABLE (fix by k2s,
+  pull request #34).
+
+* added local tox tests (added by shawnbrown, pull request #54.)
+
+* tests were broken by incorrect use of fixtures (fix by Stanislav
+  Levin, pull request #33.)
+
+
 2.0.7 - 2016-11-24
 ^^^^^^^^^^^^^^^^^^
 

docs/dbf_objects.rst

@@ -6,7 +6,7 @@ Arguments
 
 filename
   The DBF file to open.
- 
+
   The file name is case insensitive, which means ``DBF('PEOPLE.DBF')``
   will open the file ``people.dbf``. If there is a memo file, it too
   will be looked for in a case insensitive manner, so
@@ -22,7 +22,7 @@ load=False
 
   You can load and unload records at any time with the ``load()`` and
   ``unload()`` methods.
- 
+
 encoding=None
   Specify character encoding to use.
 
@@ -45,7 +45,7 @@ lowernames=False
   Field names are typically uppercase. If you pass ``True`` all field
   names will be converted to lowercase.
 
-recfactory=collections.OrderedDict
+recfactory=dict (collections.OrderedDict in older versions of Python)
 
   Takes a function that will be used to produce new records. The
   function will be called with a list of ``(name, value)`` pairs.

docs/index.rst

@@ -6,7 +6,7 @@ dbfread - Read DBF Files with Python
 ====================================
 
 Version |version|
- 
+
 DBF is a file format used by databases such dBase, Visual FoxPro, and
 FoxBase+. This library reads DBF files and returns the data as native
 Python data types for further processing. It is primarily intended for
@@ -18,8 +18,9 @@ batch jobs and one-off scripts.
     >>> from dbfread import DBF
     >>> for record in DBF('people.dbf'):
     ...     print(record)
-    OrderedDict([('NAME', 'Alice'), ('BIRTHDATE', datetime.date(1987, 3, 1))])
-    OrderedDict([('NAME', 'Bob'), ('BIRTHDATE', datetime.date(1980, 11, 12))])
+    {'NAME': 'Alice', 'BIRTHDATE': datetime.date(1987, 3, 1)}
+    {'NAME': 'Bob', 'BIRTHDATE': datetime.date(1980, 11, 12)}
+
 
 
 Source code
@@ -35,7 +36,7 @@ This document is available at https://dbfread.readthedocs.io/
 
 To build documentation locally::
 
-    python setup.py docs                                                        
+    python setup.py docs
 
 This requires Sphinx. The resulting files can be found in
 ``docs/_build/``.

docs/installing.rst

@@ -4,7 +4,7 @@ Installing dbfread
 Requirements
 ------------
 
-Requires Python 3.2 or 2.7.  ``dbfread`` is a pure Python module, so
+Requires Python 3.6 or 2.7.  ``dbfread`` is a pure Python module, so
 doesn't depend on any packages outside the standard library.
 
 

docs/introduction.rst

@@ -10,33 +10,46 @@ you can find ``people.dbf`` in ``examples/files/``.
 Opening a DBF File
 ------------------
 
-::
+.. code-block:: python
 
     >>> from dbfread import DBF
     >>> table = DBF('people.dbf')
 
-This returns a ``DBF`` object. You can now iterate over records::
+This returns a ``DBF`` object. You can now iterate over records:
+
+.. code-block:: python
 
     >>> for record in table:
     ...     print(record)
-    OrderedDict([('NAME', 'Alice'), ('BIRTHDATE', datetime.date(1987, 3, 1))])
-    OrderedDict([('NAME', 'Bob'), ('BIRTHDATE', datetime.date(1980, 11, 12))])
+    {'NAME': 'Alice', 'BIRTHDATE': datetime.date(1987, 3, 1)}
+    {'NAME', 'Bob', 'BIRTHDATE': datetime.date(1980, 11, 12)}
+
+.. note:: In older versions where dictionaries are not ordered you will
+          instead get a ``collections.OrderedDict``.
+
+Records will be returned in the order they appear in the file.
 
-and count records::
+You can count records:
+
+.. code-block:: python
 
     >>> len(table)
     2
 
-Deleted records are available in ``deleted``::
+Deleted records are available in ``deleted``:
+
+.. code-block:: python
 
     >>> for record in table.deleted:
     ...     print(record)
-    OrderedDict([('NAME', 'Deleted Guy'), ('BIRTHDATE', datetime.date(1979, 12, 22))])
+    {'NAME': 'Deleted Guy', 'BIRTHDATE': datetime.date(1979, 12, 22)}
 
     >>> len(table.deleted)
     1
 
-You can also use the ``with`` statement::
+You can also use the ``with`` statement:
+
+.. code-block:: python
 
     with DBF('people.dbf') as table:
         ...
@@ -52,7 +65,9 @@ By default records are streamed directly off disk, which means only
 one record is in memory at a time.
 
 If you have enough memory, you can load the records into a list by passing
-``load=True``. This allows for random access::
+``load=True``. This allows for random access:
+
+.. code-block:: python
 
     >>> table = DBF('people.dbf', load=True)
     >>> print(table.records[1]['NAME'])
@@ -69,7 +84,9 @@ function which returns a list of tables in a directory and load only
 the ones you need.
 
 If you just want a list of records and you don't care about the other
-table attributes you can do::
+table attributes you can do:
+
+.. code-block:: python
 
     >>> records = list(DBF('people.dbf'))
 
@@ -139,7 +156,9 @@ factory.
 A record factory is a function that takes a list of ``(name, value)``
 pairs and returns a record.  You can do whatever you like with this
 data. Here's a function that creates a record object with fields as
-attributes::
+attributes:
+
+.. code-block:: python
 
     class Record(object):
         def __init__(self, items):
@@ -159,7 +178,9 @@ Custom Field Types
 
 If the included message types are not enough you can add your own by
 subclassing ``FieldParser``. As a silly example, here how you can read
-text (``C``) fields in reverse::
+text (``C``) fields in reverse:
+
+.. code-block:: python
 
     from dbfread import DBF, FieldParser
 
@@ -172,7 +193,9 @@ text (``C``) fields in reverse::
         print(record['NAME'])
 
 and here's how you can return invalid values as ``InvalidValue``
-instead of raising ``ValueError``::
+instead of raising ``ValueError``:
+
+.. code-block:: python
 
     from dbfread import DBF, FieldParser, InvalidValue
 

examples/dbf2sqlite

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-
 """
 dbf2sqlite - convert dbf files into sqlite database
 
@@ -15,8 +14,6 @@ Todo:
 - insert only option?
 - options to select columns to insert?
 """
-
-import os
 import sys
 import argparse
 import sqlite3
@@ -40,27 +37,28 @@ typemap = {
 def add_table(cursor, table):
     """Add a dbase table to an open sqlite database."""
 
-    cursor.execute('drop table if exists %s' % table.name)
+    cursor.execute('drop table if exists "{}"'.format(table.name))
 
     field_types = {}
-    for f in table.fields:
-        field_types[f.name] = typemap.get(f.type, 'TEXT')
+    for field in table.fields:
+        field_types[field.name] = typemap.get(field.type, 'TEXT')
 
     #
     # Create the table
     #
-    defs = ', '.join(['"%s" %s' % (f, field_types[f])
+    defs = ', '.join(['"{}" {}'.format(f, field_types[f])
                       for f in table.field_names])
-    sql = 'create table "%s" (%s)' % (table.name, defs)
+    sql = 'create table "{}" ({})'.format(table.name, defs)
     cursor.execute(sql)
 
     # Create data rows
-    refs = ', '.join([':' + f for f in table.field_names])
-    sql = 'insert into "%s" values (%s)' % (table.name, refs)
+    refs = ', '.join([':' + field for field in table.field_names])
+    sql = 'insert into "{}" values ({})'.format(table.name, refs)
 
     for rec in table:
         cursor.execute(sql, list(rec.values()))
 
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description='usage: %prog [OPTIONS] table1.dbf ... tableN.dbf')
@@ -92,6 +90,7 @@ def parse_args():
 
     return parser.parse_args()
 
+
 def main():
     args = parse_args()
 
@@ -104,7 +103,7 @@ def main():
                                   lowernames=True,
                                   encoding=args.encoding,
                                   char_decode_errors=args.char_decode_errors))
-        except UnicodeDecodeError as err:
+        except UnicodeDecodeError:
             traceback.print_exc()
             sys.exit('Please use --encoding or --char-decode-errors.')
 
@@ -122,5 +121,6 @@ def main():
         for line in conn.iterdump():
             print(line)
 
+
 if __name__ == '__main__':
     main()

setup.py

@@ -56,5 +56,8 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
 )