diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index dd0890f..13c9b2d 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -69,7 +69,7 @@ CONFIG = {} -ROWS_PER_NETWORK_CALL = 70_000 +ROWS_PER_NETWORK_CALL = 40_000 def discover_catalog(conn, db_schema): @@ -373,13 +373,12 @@ def sync_table(connection, catalog_entry, state): cursor.itersize = ROWS_PER_NETWORK_CALL cursor.execute(select, params) - row = cursor.fetchone() rows_saved = 0 with metrics.record_counter(None) as counter: counter.tags['database'] = catalog_entry.database counter.tags['table'] = catalog_entry.table - while row: + for row in cursor: counter.increment() rows_saved += 1 record_message = row_to_record(catalog_entry, @@ -397,7 +396,6 @@ def sync_table(connection, catalog_entry, state): replication_key]) if rows_saved % 1000 == 0: yield singer.StateMessage(value=copy.deepcopy(state)) - row = cursor.fetchone() if not replication_key: yield activate_version_message