diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 59c8a2323..355f27147 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,12 +9,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 930396564..788df25c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,13 +10,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c51b10178..767b170ec 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -953,14 +953,16 @@ def insert_upsert_implementation( decoded = io.TextIOWrapper(file, encoding=encoding) tracker = None - if csv or tsv: - if sniff: - # Read first 2048 bytes and use that to detect - first_bytes = sniff_buffer.peek(2048) - dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore")) - else: - dialect = "excel-tab" if tsv else "excel" - with file_progress(decoded, silent=silent) as decoded: + with file_progress(decoded, silent=silent) as decoded: + if csv or tsv: + if sniff: + # Read first 2048 bytes and use that to detect + first_bytes = sniff_buffer.peek(2048) + dialect = csv_std.Sniffer().sniff( + first_bytes.decode(encoding, "ignore") + ) + else: + dialect = "excel-tab" if tsv else "excel" csv_reader_args = {"dialect": dialect} if delimiter: csv_reader_args["delimiter"] = delimiter @@ -977,24 +979,24 @@ def insert_upsert_implementation( if detect_types: tracker = TypeTracker() docs = tracker.wrap(docs) - elif lines: - docs = ({"line": line.strip()} for line in decoded) - elif text: - docs = ({"text": decoded.read()},) - else: - try: - if nl: - docs = (json.loads(line) for line in decoded if line.strip()) - else: - docs = json.load(decoded) - if isinstance(docs, dict): - docs = [docs] - except json.decoder.JSONDecodeError: - raise click.ClickException( - "Invalid JSON - use --csv for CSV or --tsv for TSV files" - ) - if flatten: - docs = (dict(_flatten(doc)) for doc in docs) + elif lines: + docs = ({"line": line.strip()} for line in decoded) + elif text: + docs = ({"text": decoded.read()},) + else: + try: + if nl: + docs = (json.loads(line) for line in decoded if line.strip()) + else: + docs = json.load(decoded) + if isinstance(docs, dict): + docs = [docs] + except json.decoder.JSONDecodeError: + raise click.ClickException( + "Invalid JSON - use --csv for CSV or --tsv for TSV files" + ) + if flatten: + docs = (dict(_flatten(doc)) for doc in docs) if convert: variable = "row" diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index c0b7bf18d..875455420 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -155,6 +155,11 @@ def __iter__(self): self._update(len(line)) yield line + def read(self, size=-1): + data = self._wrapped.read(size) + self._update(len(data)) + return data + @contextlib.contextmanager def file_progress(file, silent=False, **kwargs):