You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
import csv
import asyncio
import aiomysql
import time
import sys
import argparse
parser = argparse.ArgumentParser(description="A python program to import and compare csv data with mysql database.")
parser.add_argument("--host", type=str, default="172.27.2.19", help="The host name of the database.")
parser.add_argument("--user", type=str, default="prog_test", help="The user name of the database.")
parser.add_argument("--password", type=str, default="abcd.1234", help="The password of the database.")
parser.add_argument("--db", type=str, default="test", help="The name of the database.")
parser.add_argument("--table", type=str, default="t1", help="The name of the table.")
parser.add_argument("action", type=str, choices=["load", "compare"], help="The action to perform: load or compare.")
load_group = parser.add_argument_group("load", "Arguments for loading csv data to database")
load_group.add_argument("--csv", type=str, required=False, help="The path of the csv file to import. Only valid for load action.")
compare_group = parser.add_argument_group("compare", "Arguments for comparing csv data with database")
compare_group.add_argument("--new_csv", type=str, required=False, help="The path of the new csv file to compare. Only valid for compare action.")
compare_group.add_argument("--output", type=str, default="output.csv", help="The path of the output csv file to write. Only valid for compare action.")
args = parser.parse_args()
DB_HOST = args.host
DB_USER = args.user
DB_PASSWORD = args.password
DB_NAME = args.db
DB_TABLE = args.table
ACTION = args.action
async def insert_batch(batch, conn):
cur = await conn.cursor()
sql = f"insert into {DB_TABLE} (id, name, identified_id) values (%s, %s, %s)"
try:
await cur.executemany(sql, batch)
await conn.commit()
except Exception as e:
print(e)
finally:
await cur.close()
async def import_csv2():
start = time.time()
pool = await aiomysql.create_pool(host=DB_HOST, user=DB_USER, password=DB_PASSWORD, db=DB_NAME)
CSV_FILE=args.csv
with open(CSV_FILE, "r", encoding="utf-8") as f:
reader = csv.reader(f)
next(reader)
tasks = []
batch = []
for row in reader:
batch.append(row)
if len(batch) == 3:
conn = await pool.acquire()
task = asyncio.create_task(insert_batch(batch, conn))
tasks.append(task)
pool.release(conn)
batch = []
if batch:
conn = await pool.acquire()
task = asyncio.create_task(insert_batch(batch, conn))
tasks.append(task)
pool.release(conn)
await asyncio.gather(*tasks)
pool.close()
await pool.wait_closed()
end = time.time()
print(f"Imported {CSV_FILE} to {DB_NAME}.{DB_TABLE} in {end - start} seconds.")
def main():
loop = asyncio.get_event_loop()
if ACTION == "load":
loop.run_until_complete(import_csv2())
else:
print(f"Invalid action: {ACTION}")
loop.close()
if __name__ == "__main__":
main()
when i run python load.py --csv load.csv load ,it throws errors:
readexactly() called while another coroutine is already waiting for incoming data
reacted with thumbs up emoji reacted with thumbs down emoji reacted with laugh emoji reacted with hooray emoji reacted with confused emoji reacted with heart emoji reacted with rocket emoji reacted with eyes emoji
-
my code is:
when i run
python load.py --csv load.csv load
,it throws errors:test.t1 table structure:
how can i do to fix it?
Beta Was this translation helpful? Give feedback.
All reactions