From cf4e6e9c97b29ad315acf04ee82a49d4f4727d46 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Tue, 22 Apr 2025 21:50:01 +0300 Subject: [PATCH 01/17] commit --- requirements.txt | 4 ++- src/amazon_api/__init__.py | 0 src/amazon_api/get_amazon_api.py | 58 ++++++++++++++++++++++++++++++ src/amazon_api/orders.py | 11 ++++++ src/configs/__init__.py | 5 +++ src/configs/env.py | 29 +++++++++------ src/main_start.py | 10 ++++++ src/parser_debug.py | 6 ++-- src/schemes/client.py | 1 + src/utils/safe_ratelimit_amazon.py | 48 +++++++++++++++++++++++++ src/utils/temporaty/check_potin.py | 11 ++++++ 11 files changed, 169 insertions(+), 14 deletions(-) create mode 100644 src/amazon_api/__init__.py create mode 100644 src/amazon_api/get_amazon_api.py create mode 100644 src/amazon_api/orders.py create mode 100644 src/main_start.py create mode 100644 src/utils/safe_ratelimit_amazon.py create mode 100644 src/utils/temporaty/check_potin.py diff --git a/requirements.txt b/requirements.txt index 5e1f809..91fac51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,6 @@ uvicorn[standard]~=0.29.0 cryptography DateTime~=5.5 -loguru~=0.7.2 \ No newline at end of file +loguru~=0.7.2 + +python-amazon-sp-api \ No newline at end of file diff --git a/src/amazon_api/__init__.py b/src/amazon_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py new file mode 100644 index 0000000..0d8f020 --- /dev/null +++ b/src/amazon_api/get_amazon_api.py @@ -0,0 +1,58 @@ +from sp_api.api import Orders +from sp_api.util import throttle_retry, load_all_pages +from datetime import datetime, timezone +from loguru import logger + +from configs import settings +from utils.safe_ratelimit_amazon import safe_rate_limit + +CREDENTIALS = dict( + lwa_app_id=settings.LWA_APP_ID, + lwa_client_secret=settings.LWA_CLIENT_SECRET +) + +LAST_UPDATE_BEFORE=datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') +#first order was offer in 2024/2/17 +CREATED_AFTER = datetime(2024, 1, 1).isoformat().replace("+00:00", 'Z') +LIMIT=100 + + + +class OrderClient: + def __init__(self): + self._order_cl = Orders(credentials=CREDENTIALS, + refresh_token=settings.SP_API_REFRESH_TOKEN) + self._expanded_orders = [] + + + @throttle_retry() + @load_all_pages() + def _load_all_orders(self, **kwargs): + return self._order_cl.get_orders(**kwargs) + + + @throttle_retry() + @load_all_pages() + @safe_rate_limit(header_limit=True) + def _load_all_items(self, order_id, **kwargs): + return self._order_cl.get_order_items(order_id=order_id, **kwargs) + + def _get_all_items(self, order_id): + items = [] + for i_page in self._load_all_items(order_id=order_id): + for item in i_page.payload.get("OrderItems"): + items.append(item) + return items + + + def get_orders(self): + for page in self._load_all_orders(CreatedAfter=CREATED_AFTER): + for order in page.payload.get('Orders'): + _order_id = order["AmazonOrderId"] + + self._expanded_orders.append({ + "Order_data": order, + "Order_item_data": self._get_all_items(order_id=_order_id) + }) + + return self._expanded_orders diff --git a/src/amazon_api/orders.py b/src/amazon_api/orders.py new file mode 100644 index 0000000..3a3b30b --- /dev/null +++ b/src/amazon_api/orders.py @@ -0,0 +1,11 @@ +from amazon_api.get_amazon_api import OrderClient + + + +def get_all_orders(): + order_client = OrderClient() + orders = order_client.get_orders() # type: ignore + + return orders + + diff --git a/src/configs/__init__.py b/src/configs/__init__.py index e69de29..f0203a5 100644 --- a/src/configs/__init__.py +++ b/src/configs/__init__.py @@ -0,0 +1,5 @@ +from .env import Settings + + + +settings = Settings() diff --git a/src/configs/env.py b/src/configs/env.py index 685a55f..4bef961 100644 --- a/src/configs/env.py +++ b/src/configs/env.py @@ -1,18 +1,25 @@ -import os +# import os -from dotenv import load_dotenv +# from dotenv import load_dotenv -load_dotenv() +# from pydantic.v1 import BaseSettings +# +# load_dotenv() +# +# LWA_APP_ID = os.getenv("LWA_APP_ID") +# LWA_CLIENT_SECRET = os.getenv("LWA_CLIENT_SECRET") +# SP_API_REFRESH_TOKEN = os.getenv("SP_API_REFRESH_TOKEN") +# -ETSY_API_KEY = os.getenv('ETSY_API_KEY') -ETSY_API_SHARED_SECRET = os.getenv('ETSY_API_SHARED_SECRET') -ETSY_API_REDIRECT_URL = os.getenv('ETSY_API_REDIRECT_URL') +from pydantic_settings import BaseSettings, SettingsConfigDict -CODE_VERIFIER = os.getenv('CODE_VERIFIER') -API_URL = os.getenv('API_URL') -API_AUTH_TOKEN = os.getenv("API_AUTH_TOKEN") +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env") + + LWA_APP_ID: str + LWA_CLIENT_SECRET: str + SP_API_REFRESH_TOKEN: str + -DATA_FOLDER_PATH = os.getenv('DATA_FOLDER_PATH') -LOG_FILE = os.getenv("LOG_FILE") diff --git a/src/main_start.py b/src/main_start.py new file mode 100644 index 0000000..70cc909 --- /dev/null +++ b/src/main_start.py @@ -0,0 +1,10 @@ +from amazon_api.orders import get_all_orders +import json + + +if __name__ == "__main__": + orders = get_all_orders() + with open("check.json", 'W') as file: + json.dump(orders, file, indent=4) + print("ok") + print(orders) \ No newline at end of file diff --git a/src/parser_debug.py b/src/parser_debug.py index 4cb5088..b538965 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -27,10 +27,11 @@ # Every 30 minutes PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 +EXCEL_FILE = "data/check_point.xlsx" + -def process_single_shop(shop): - now_hour = datetime.now().hour +def process_single_shop(shop): shop_error = False start_time_shop = datetime.now() @@ -47,6 +48,7 @@ def process_single_shop(shop): that_month = True offset = 0 date = datetime.now() - timedelta(days=30) + count = 0 # for excel check_point weekday = datetime.now().weekday() ########################## diff --git a/src/schemes/client.py b/src/schemes/client.py index d67e068..327bfcc 100644 --- a/src/schemes/client.py +++ b/src/schemes/client.py @@ -5,3 +5,4 @@ class Client(BaseModel): user_marketplace_id: str | None = None name: str | None = None email: str | None = None + diff --git a/src/utils/safe_ratelimit_amazon.py b/src/utils/safe_ratelimit_amazon.py new file mode 100644 index 0000000..81bcfa8 --- /dev/null +++ b/src/utils/safe_ratelimit_amazon.py @@ -0,0 +1,48 @@ +from sp_api.base import ApiResponse +import time +from loguru import logger + + +def _delay_execution(*, + throttle_by_seconds: int, + header_limit: bool, + rate_limit: float) -> float: + """delay in seconds""" + if header_limit and rate_limit: + return 1 / float(rate_limit) # for dynamically rate limit + return float(throttle_by_seconds) + +def _check_next_page(): + + + +def safe_rate_limit(throttle_by_seconds: int = 1, + header_limit: bool = False): + def decorator(function): + def wrapper(*args, **kwargs): + resp: ApiResponse = function(*args, **kwargs) + if not isinstance(resp, ApiResponse): + return resp + if resp.next_token: + #excludes delay for several pages + return resp + logger.info(resp) + logger.info(f"в декораторе и прошёл первичную проверку....") + + sleep_time = _delay_execution(throttle_by_seconds=throttle_by_seconds, + header_limit=header_limit, + rate_limit=resp.rate_limit) + + if sleep_time: + logger.info(f"запуск слипа....{sleep_time}") + time.sleep(sleep_time) + return resp + + wrapper.__doc__ = function.__doc__ + return wrapper + + return decorator + + + + diff --git a/src/utils/temporaty/check_potin.py b/src/utils/temporaty/check_potin.py new file mode 100644 index 0000000..64e832b --- /dev/null +++ b/src/utils/temporaty/check_potin.py @@ -0,0 +1,11 @@ +import openpyxl + + +def save_data_row(*, + data: str, f_name: str, count: int = 0): + """data is id row""" + wb = openpyxl.Workbook() + sheet = wb.active + sheet.cell(row=count, column=1).value = data + + wb.save(f_name) From 4fc75ba85bfa64ad4a2f070160f56584a9fbf1ff Mon Sep 17 00:00:00 2001 From: Trydimas Date: Wed, 23 Apr 2025 14:27:55 +0300 Subject: [PATCH 02/17] excel checkpoint --- src/configs/env.py | 1 + src/constants/files_paths.py | 2 ++ src/parser_debug.py | 38 ++++++++++++++---------------- src/schemes/shop_data.py | 8 +------ src/utils/excel/__init__.py | 0 src/utils/excel/write_each_row.py | 21 +++++++++++++++++ src/utils/temporaty/check_potin.py | 11 --------- 7 files changed, 43 insertions(+), 38 deletions(-) create mode 100644 src/utils/excel/__init__.py create mode 100644 src/utils/excel/write_each_row.py delete mode 100644 src/utils/temporaty/check_potin.py diff --git a/src/configs/env.py b/src/configs/env.py index 4bef961..7fe1cee 100644 --- a/src/configs/env.py +++ b/src/configs/env.py @@ -21,5 +21,6 @@ class Settings(BaseSettings): LWA_APP_ID: str LWA_CLIENT_SECRET: str SP_API_REFRESH_TOKEN: str + LOG_FILE: str diff --git a/src/constants/files_paths.py b/src/constants/files_paths.py index 7e16aff..67dd717 100644 --- a/src/constants/files_paths.py +++ b/src/constants/files_paths.py @@ -1,3 +1,5 @@ from configs.env import DATA_FOLDER_PATH SHOPS_DATA_FILE_PATH = f"{DATA_FOLDER_PATH}/shops/shops.json" + + diff --git a/src/parser_debug.py b/src/parser_debug.py index b538965..c0b7dc8 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -1,22 +1,21 @@ import concurrent.futures import json import pprint -import time from datetime import datetime, timedelta from loguru import logger as log -from api.order import upload_orders_data from api.parser import update_parser_status_by_id -from configs.env import LOG_FILE from constants.status import ParserStatus from etsy_api.orders import get_all_orders_by_shop_id from schemes.upload_order import UploadingOrderData, OrderData from utils.format_order_data import format_order_data from utils.parser_shops_data import get_parser_shops_data +from utils.excel.write_each_row import write_to_excel +from configs import settings log.add( - LOG_FILE, + settings.LOG_FILE, format="{time} {level} {message}", level="DEBUG", rotation="100 MB", @@ -26,17 +25,28 @@ # Every 30 minutes PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 +CREATED_AFTER = datetime(2024, 1, 1).isoformat().replace("+00:00", 'Z') -EXCEL_FILE = "data/check_point.xlsx" +EXCEL_FILE = "data/check_point.xlsx" #temporaty def process_single_shop(shop): - shop_error = False + # Initializing constants + shop_error = False start_time_shop = datetime.now() + + offset = 0 + date = datetime.now() - timedelta(days=30) # will using in CREATED_AFTER flag + # weekday = datetime.now().weekday() + ########################## + log.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") - log.info(f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}...") + log.info( + f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." + ) + update_parser_status_by_id( parser_id=shop.parser_id, status=ParserStatus.PARSING, @@ -44,14 +54,6 @@ def process_single_shop(shop): log.success(f"Parser status updated.") - # Initializing constants - that_month = True - offset = 0 - date = datetime.now() - timedelta(days=30) - count = 0 # for excel check_point - weekday = datetime.now().weekday() - ########################## - while that_month: uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) @@ -81,9 +83,6 @@ def process_single_shop(shop): order, goods_in_order, day, month, client, city = format_order_data( order=shop_order, ) - if day <= date.day and month == date.month: - that_month = False - break uploading_orders.orders_data.append( OrderData( @@ -98,7 +97,6 @@ def process_single_shop(shop): with open("test_data.json", "w") as f: json.dump(uploading_orders.model_dump(), f) - return offset += 100 @@ -146,4 +144,4 @@ def etsy_api_parser(): if __name__ == "__main__": shops_data = get_parser_shops_data() - process_single_shop(shops_data[1]) + process_single_shop(shops_data[2]) diff --git a/src/schemes/shop_data.py b/src/schemes/shop_data.py index 168b457..2cd73f0 100644 --- a/src/schemes/shop_data.py +++ b/src/schemes/shop_data.py @@ -5,10 +5,4 @@ class ShopData(BaseModel): parser_id: int shop_id: int shop_name: str - # TODO delete param shop cookie - shop_cookie: str - shop_token: str - shop_refresh_token: str - expiry: float - etsy_shop_id: str - shop_auth_code: str + amazon_shop_id: str diff --git a/src/utils/excel/__init__.py b/src/utils/excel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils/excel/write_each_row.py b/src/utils/excel/write_each_row.py new file mode 100644 index 0000000..08ad44e --- /dev/null +++ b/src/utils/excel/write_each_row.py @@ -0,0 +1,21 @@ +import openpyxl +from openpyxl import load_workbook +import os + + +def write_to_excel(value, filename='output.xlsx'): + if os.path.exists(filename): + wb = load_workbook(filename) + else: + wb = openpyxl.Workbook() + + sheet = wb.active + + #finde first empty row in cell A + next_row = sheet.max_row + 1 + while sheet.cell(row=next_row, column=1).value is not None: + next_row += 1 + + sheet.cell(row=next_row, column=1).value = value + + wb.save(filename) \ No newline at end of file diff --git a/src/utils/temporaty/check_potin.py b/src/utils/temporaty/check_potin.py deleted file mode 100644 index 64e832b..0000000 --- a/src/utils/temporaty/check_potin.py +++ /dev/null @@ -1,11 +0,0 @@ -import openpyxl - - -def save_data_row(*, - data: str, f_name: str, count: int = 0): - """data is id row""" - wb = openpyxl.Workbook() - sheet = wb.active - sheet.cell(row=count, column=1).value = data - - wb.save(f_name) From e3b20d2245e3a707123879c080f04af40cf05db4 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Wed, 23 Apr 2025 15:44:06 +0300 Subject: [PATCH 03/17] add constants and orders refactor --- src/amazon_api/get_amazon_api.py | 33 +++++++++++++++++------------ src/constants/amazon_credentials.py | 9 ++++++++ src/constants/amazon_dates.py | 12 +++++++++++ src/utils/format_datetime.py | 10 +++++++++ 4 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 src/constants/amazon_credentials.py create mode 100644 src/constants/amazon_dates.py create mode 100644 src/utils/format_datetime.py diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py index 0d8f020..c851c2d 100644 --- a/src/amazon_api/get_amazon_api.py +++ b/src/amazon_api/get_amazon_api.py @@ -1,28 +1,24 @@ from sp_api.api import Orders from sp_api.util import throttle_retry, load_all_pages -from datetime import datetime, timezone -from loguru import logger from configs import settings +from constants.amazon_credentials import CREDENTIALS_ARG +from constants.amazon_dates import LAST_MONTH from utils.safe_ratelimit_amazon import safe_rate_limit +from utils.format_datetime import is_iso_utc_z_format -CREDENTIALS = dict( - lwa_app_id=settings.LWA_APP_ID, - lwa_client_secret=settings.LWA_CLIENT_SECRET -) -LAST_UPDATE_BEFORE=datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') -#first order was offer in 2024/2/17 -CREATED_AFTER = datetime(2024, 1, 1).isoformat().replace("+00:00", 'Z') LIMIT=100 class OrderClient: - def __init__(self): - self._order_cl = Orders(credentials=CREDENTIALS, - refresh_token=settings.SP_API_REFRESH_TOKEN) + def __init__(self, log): + self._order_cl = Orders() self._expanded_orders = [] + self.log = log + + @throttle_retry() @@ -45,8 +41,17 @@ def _get_all_items(self, order_id): return items - def get_orders(self): - for page in self._load_all_orders(CreatedAfter=CREATED_AFTER): + def get_orders(self, created_after: str): + """ + Args: + created_after: iso6108 format with Z + """ + + if not is_iso_utc_z_format(created_after): + created_after = LAST_MONTH + + for page in self._load_all_orders(CreatedAfter=created_after): + for order in page.payload.get('Orders'): _order_id = order["AmazonOrderId"] diff --git a/src/constants/amazon_credentials.py b/src/constants/amazon_credentials.py new file mode 100644 index 0000000..73e09e8 --- /dev/null +++ b/src/constants/amazon_credentials.py @@ -0,0 +1,9 @@ +from configs import settings + +CREDENTIALS_ARG = dict( + refresh_token=settings.SP_API_REFRESH_TOKEN, + credentials=dict( + lwa_app_id=settings.LWA_APP_ID, + lwa_client_secret=settings.LWA_CLIENT_SECRET + ) +) diff --git a/src/constants/amazon_dates.py b/src/constants/amazon_dates.py new file mode 100644 index 0000000..7abbc6e --- /dev/null +++ b/src/constants/amazon_dates.py @@ -0,0 +1,12 @@ +from datetime import datetime, timezone, timedelta + +# WARNING: only ISO8601, endswith == Z + +#first order was offer in 2024/2/17 +EARLIEST_DATE = datetime(2024, 1, 1, tzinfo=timezone.utc).isoformat().replace("+00:00", 'Z') + +#current date +CURRENT_DATE=datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') + +#last month date +LAST_MONTH = (datetime.now(timezone.utc) - timedelta(30)).isoformat().replace("+00:00", "Z") \ No newline at end of file diff --git a/src/utils/format_datetime.py b/src/utils/format_datetime.py new file mode 100644 index 0000000..5e2035a --- /dev/null +++ b/src/utils/format_datetime.py @@ -0,0 +1,10 @@ +from datetime import datetime + +def is_iso_utc_z_format(date_str): + if not date_str.endswith('Z'): + return False + try: + dt = datetime.fromisoformat(date_str[:-1] + '+00:00') + return dt.utcoffset().total_seconds() == 0 + except ValueError: + return False From dd96e91ac091db826e4d7bfad56c137c931568cc Mon Sep 17 00:00:00 2001 From: Trydimas Date: Wed, 23 Apr 2025 16:45:27 +0300 Subject: [PATCH 04/17] tmp: amazon dates constants and refactor parser debug and get amazon api --- src/amazon_api/get_amazon_api.py | 26 ++++--------- src/constants/amazon_dates.py | 2 +- src/log/logger.py | 14 +++++++ src/parser_debug.py | 66 ++++++++++++++++++++++---------- 4 files changed, 68 insertions(+), 40 deletions(-) create mode 100644 src/log/logger.py diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py index c851c2d..f808f0a 100644 --- a/src/amazon_api/get_amazon_api.py +++ b/src/amazon_api/get_amazon_api.py @@ -1,17 +1,14 @@ from sp_api.api import Orders from sp_api.util import throttle_retry, load_all_pages +from sp_api.base import ApiResponse from configs import settings from constants.amazon_credentials import CREDENTIALS_ARG -from constants.amazon_dates import LAST_MONTH +from constants.amazon_dates import LAST_MONTH_DATE from utils.safe_ratelimit_amazon import safe_rate_limit from utils.format_datetime import is_iso_utc_z_format -LIMIT=100 - - - class OrderClient: def __init__(self, log): self._order_cl = Orders() @@ -19,11 +16,9 @@ def __init__(self, log): self.log = log - - @throttle_retry() @load_all_pages() - def _load_all_orders(self, **kwargs): + def load_all_orders(self, **kwargs): return self._order_cl.get_orders(**kwargs) @@ -33,6 +28,7 @@ def _load_all_orders(self, **kwargs): def _load_all_items(self, order_id, **kwargs): return self._order_cl.get_order_items(order_id=order_id, **kwargs) + def _get_all_items(self, order_id): items = [] for i_page in self._load_all_items(order_id=order_id): @@ -41,17 +37,9 @@ def _get_all_items(self, order_id): return items - def get_orders(self, created_after: str): - """ - Args: - created_after: iso6108 format with Z - """ - - if not is_iso_utc_z_format(created_after): - created_after = LAST_MONTH - - for page in self._load_all_orders(CreatedAfter=created_after): + def get_orders(self, page: ApiResponse): + try: for order in page.payload.get('Orders'): _order_id = order["AmazonOrderId"] @@ -59,5 +47,7 @@ def get_orders(self, created_after: str): "Order_data": order, "Order_item_data": self._get_all_items(order_id=_order_id) }) + except Exception as e: + self.log. return self._expanded_orders diff --git a/src/constants/amazon_dates.py b/src/constants/amazon_dates.py index 7abbc6e..41d7e61 100644 --- a/src/constants/amazon_dates.py +++ b/src/constants/amazon_dates.py @@ -9,4 +9,4 @@ CURRENT_DATE=datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') #last month date -LAST_MONTH = (datetime.now(timezone.utc) - timedelta(30)).isoformat().replace("+00:00", "Z") \ No newline at end of file +LAST_MONTH_DATE = (datetime.now(timezone.utc) - timedelta(30)).isoformat().replace("+00:00", "Z") \ No newline at end of file diff --git a/src/log/logger.py b/src/log/logger.py new file mode 100644 index 0000000..a037a17 --- /dev/null +++ b/src/log/logger.py @@ -0,0 +1,14 @@ +from loguru import logger +from configs import settings + +logger.add( + settings.LOG_FILE, + format="{time} {level} {message}", + level="DEBUG", + rotation="100 MB", + compression="zip", + serialize=True, +) + + +__all__ = ["logger"] diff --git a/src/parser_debug.py b/src/parser_debug.py index c0b7dc8..1706fdd 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -3,8 +3,6 @@ import pprint from datetime import datetime, timedelta -from loguru import logger as log - from api.parser import update_parser_status_by_id from constants.status import ParserStatus from etsy_api.orders import get_all_orders_by_shop_id @@ -13,6 +11,19 @@ from utils.parser_shops_data import get_parser_shops_data from utils.excel.write_each_row import write_to_excel from configs import settings +from amazon_api.get_amazon_api import OrderClient + + + +from sp_api.util import throttle_retry, load_all_pages + +from configs import settings +from constants.amazon_credentials import CREDENTIALS_ARG +from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE +from utils.safe_ratelimit_amazon import safe_rate_limit +from utils.format_datetime import is_iso_utc_z_format +from log.logger import logger + log.add( settings.LOG_FILE, @@ -30,20 +41,22 @@ EXCEL_FILE = "data/check_point.xlsx" #temporaty +#TODO link on update refresh token don't forget!!!! def process_single_shop(shop): # Initializing constants shop_error = False start_time_shop = datetime.now() + order_cl = OrderClient(log=logger) offset = 0 - date = datetime.now() - timedelta(days=30) # will using in CREATED_AFTER flag + created_after = EARLIEST_DATE # weekday = datetime.now().weekday() ########################## - log.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") - log.info( + logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") + logger.info( f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." ) @@ -52,23 +65,30 @@ def process_single_shop(shop): status=ParserStatus.PARSING, ) - log.success(f"Parser status updated.") + logger.success(f"Parser status updated.") + - while that_month: - uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) + uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) - log.info( + + #loop + for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): + """Every 100 orders after """ + + logger.info( f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." ) + + try: shop_orders, _ = get_all_orders_by_shop_id( - etsy_shop_id=int(shop.etsy_shop_id), - shop_id=shop.shop_id, - limit=100, - offset=offset, + etsy_shop_id=int(shop.etsy_shop_id), + shop_id=shop.shop_id, + limit=100, + offset=offset, ) except Exception as e: - log.critical(f"Some error in getting info from ETSY API: {e}") + logger.critical(f"Some error in getting info from ETSY API: {e}") pprint.pprint(e) update_parser_status_by_id( parser_id=shop.parser_id, @@ -77,6 +97,10 @@ def process_single_shop(shop): shop_error = True break + + + + # Get order details and split for creating and updating for shop_order in shop_orders: @@ -106,9 +130,9 @@ def process_single_shop(shop): break if shop_error: - log.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") + logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") return - log.info( + logger.info( f"Updating parser {shop.parser_id} status to {ParserStatus.OK_AND_WAIT}..." ) @@ -118,10 +142,10 @@ def process_single_shop(shop): last_parsed=datetime.now().timestamp(), ) - log.success(f"Parser status updated.") - log.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") + logger.success(f"Parser status updated.") + logger.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") end_time_shop = datetime.now() - log.info( + logger.info( f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" ) @@ -138,8 +162,8 @@ def etsy_api_parser(): # Проверяем, были ли исключения for future in futures: if future.exception(): - log.error(f"Error in thread: {future.exception()}") - log.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") + logger.error(f"Error in thread: {future.exception()}") + logger.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") if __name__ == "__main__": From 9a3afce180d367777ad8b91ce44e57183fd483ae Mon Sep 17 00:00:00 2001 From: Trydimas Date: Wed, 23 Apr 2025 17:57:26 +0300 Subject: [PATCH 05/17] mini refactor --- src/api/auth.py | 6 ++++-- src/api/parser.py | 13 ++++++++----- src/configs/env.py | 4 ++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/api/auth.py b/src/api/auth.py index 5920058..ef36f62 100644 --- a/src/api/auth.py +++ b/src/api/auth.py @@ -1,8 +1,10 @@ -from configs.env import API_AUTH_TOKEN +from configs import settings from schemes.auth import Auth def authorization() -> Auth: return Auth( - Authorization=f"Bearer {API_AUTH_TOKEN}" + Authorization=f"Bearer {settings.API_AUTH_TOKEN}" ) + + diff --git a/src/api/parser.py b/src/api/parser.py index 03b2e15..642aa87 100644 --- a/src/api/parser.py +++ b/src/api/parser.py @@ -1,27 +1,30 @@ import requests as req -from loguru import logger as log +from log.logger import logger from api.auth import authorization -from configs.env import API_URL +from configs import settings def update_parser_status_by_id( - parser_id: int, status: int, last_parsed: float | None = None + parser_id: int, status: int, last_parsed: float | None = None ): data = { "id": parser_id, "status": status, } + if last_parsed: data.update({"last_parsed": last_parsed}) + try: response = req.put( - f"{API_URL}/parser/", headers=authorization().model_dump(), json=data + f"{settings.API_URL}/parser/", headers=authorization().model_dump(), json=data ) except Exception as e: return update_parser_status_by_id(parser_id, status) + if response.status_code != 200: - log.error( + logger.error( f""" Some error when updating parser status. Parser ID: {parser_id} diff --git a/src/configs/env.py b/src/configs/env.py index 7fe1cee..cc2479a 100644 --- a/src/configs/env.py +++ b/src/configs/env.py @@ -21,6 +21,10 @@ class Settings(BaseSettings): LWA_APP_ID: str LWA_CLIENT_SECRET: str SP_API_REFRESH_TOKEN: str + + API_URL: str + API_AUTH_TOKEN: str + LOG_FILE: str From 0292ba8189a2fbf1706f3b2a3379342085c04462 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Wed, 23 Apr 2025 17:57:48 +0300 Subject: [PATCH 06/17] try/ex migrate --- src/amazon_api/get_amazon_api.py | 37 +++++++++++++++++++++----------- src/parser_debug.py | 9 +------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py index f808f0a..0dfdc7e 100644 --- a/src/amazon_api/get_amazon_api.py +++ b/src/amazon_api/get_amazon_api.py @@ -1,44 +1,49 @@ from sp_api.api import Orders from sp_api.util import throttle_retry, load_all_pages from sp_api.base import ApiResponse +import pprint from configs import settings from constants.amazon_credentials import CREDENTIALS_ARG from constants.amazon_dates import LAST_MONTH_DATE +from constants.status import ParserStatus from utils.safe_ratelimit_amazon import safe_rate_limit from utils.format_datetime import is_iso_utc_z_format +from api.parser import update_parser_status_by_id +from log.logger import logger +from schemes.shop_data import ShopData class OrderClient: - def __init__(self, log): - self._order_cl = Orders() + def __init__(self, shop: ShopData): self._expanded_orders = [] - self.log = log + self.shop = shop + + def _get_all_items(self, order_id): + items = [] + for i_page in self._load_all_items(order_id=order_id): + for item in i_page.payload.get("OrderItems"): + items.append(item) + return items @throttle_retry() @load_all_pages() def load_all_orders(self, **kwargs): - return self._order_cl.get_orders(**kwargs) + return self.get_orders(**kwargs) @throttle_retry() @load_all_pages() @safe_rate_limit(header_limit=True) def _load_all_items(self, order_id, **kwargs): - return self._order_cl.get_order_items(order_id=order_id, **kwargs) + return self._get_order_items(order_id=order_id, **kwargs) - def _get_all_items(self, order_id): - items = [] - for i_page in self._load_all_items(order_id=order_id): - for item in i_page.payload.get("OrderItems"): - items.append(item) - return items - def get_orders(self, page: ApiResponse): + def get_orders(self, page: ApiResponse): try: for order in page.payload.get('Orders'): _order_id = order["AmazonOrderId"] @@ -48,6 +53,12 @@ def get_orders(self, page: ApiResponse): "Order_item_data": self._get_all_items(order_id=_order_id) }) except Exception as e: - self.log. + logger.critical(f"Some error in getting info from Amazon SP API: {e}") + pprint.pprint(e) + update_parser_status_by_id( + parser_id=self.shop.parser_id, + status=ParserStatus.OK_AND_WAIT + ) + #TODO: retrurn shop_error return self._expanded_orders diff --git a/src/parser_debug.py b/src/parser_debug.py index 1706fdd..f7fb6a5 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -25,14 +25,6 @@ from log.logger import logger -log.add( - settings.LOG_FILE, - format="{time} {level} {message}", - level="DEBUG", - rotation="100 MB", - compression="zip", - serialize=True, -) # Every 30 minutes PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 @@ -80,6 +72,7 @@ def process_single_shop(shop): ) + #TODO make try ex try: shop_orders, _ = get_all_orders_by_shop_id( etsy_shop_id=int(shop.etsy_shop_id), From d071b6cec23fd0df6e6bef16f53dce0f0cf84985 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Thu, 24 Apr 2025 03:25:07 +0300 Subject: [PATCH 07/17] complete refactor parse orders_data and compressed try/except --- src/amazon_api/get_amazon_api.py | 33 +++--- src/amazon_api/orders.py | 2 +- src/parser_debug.py | 43 ++------ src/utils/format_datetime.py | 7 ++ src/utils/format_order_data.py | 173 ++++++++++++------------------- 5 files changed, 101 insertions(+), 157 deletions(-) diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py index 0dfdc7e..1731c4e 100644 --- a/src/amazon_api/get_amazon_api.py +++ b/src/amazon_api/get_amazon_api.py @@ -5,21 +5,23 @@ from configs import settings from constants.amazon_credentials import CREDENTIALS_ARG -from constants.amazon_dates import LAST_MONTH_DATE from constants.status import ParserStatus from utils.safe_ratelimit_amazon import safe_rate_limit -from utils.format_datetime import is_iso_utc_z_format from api.parser import update_parser_status_by_id from log.logger import logger from schemes.shop_data import ShopData +from schemes.upload_order import OrderData +from utils.format_order_data import format_order_data class OrderClient: def __init__(self, shop: ShopData): - self._expanded_orders = [] + self.order_api = Orders(**CREDENTIALS_ARG) + self._list_orders_data = [] self.shop = shop - def _get_all_items(self, order_id): + + def _get_all_items(self, order_id) -> list: items = [] for i_page in self._load_all_items(order_id=order_id): for item in i_page.payload.get("OrderItems"): @@ -30,28 +32,28 @@ def _get_all_items(self, order_id): @throttle_retry() @load_all_pages() def load_all_orders(self, **kwargs): - return self.get_orders(**kwargs) + return self.order_api.get_orders(**kwargs) @throttle_retry() @load_all_pages() @safe_rate_limit(header_limit=True) def _load_all_items(self, order_id, **kwargs): - return self._get_order_items(order_id=order_id, **kwargs) - + return self.order_api.get_order_items(order_id=order_id, **kwargs) +#TODO format orders and items - def get_orders(self, page: ApiResponse): + def get_orders_with_items(self, page: ApiResponse) -> list[OrderData] | None: try: for order in page.payload.get('Orders'): _order_id = order["AmazonOrderId"] - - self._expanded_orders.append({ - "Order_data": order, - "Order_item_data": self._get_all_items(order_id=_order_id) - }) + order_data = format_order_data( + order=order, + items=self._get_all_items(order_id=_order_id) + ) + self._list_orders_data.append(order_data) except Exception as e: logger.critical(f"Some error in getting info from Amazon SP API: {e}") pprint.pprint(e) @@ -59,6 +61,5 @@ def get_orders(self, page: ApiResponse): parser_id=self.shop.parser_id, status=ParserStatus.OK_AND_WAIT ) - #TODO: retrurn shop_error - - return self._expanded_orders + return None + return self._list_orders_data diff --git a/src/amazon_api/orders.py b/src/amazon_api/orders.py index 3a3b30b..0ef14be 100644 --- a/src/amazon_api/orders.py +++ b/src/amazon_api/orders.py @@ -4,7 +4,7 @@ def get_all_orders(): order_client = OrderClient() - orders = order_client.get_orders() # type: ignore + orders = order_client.get_orders_with_items() # type: ignore return orders diff --git a/src/parser_debug.py b/src/parser_debug.py index f7fb6a5..1561a66 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -72,43 +72,20 @@ def process_single_shop(shop): ) - #TODO make try ex - try: - shop_orders, _ = get_all_orders_by_shop_id( - etsy_shop_id=int(shop.etsy_shop_id), - shop_id=shop.shop_id, - limit=100, - offset=offset, - ) - except Exception as e: - logger.critical(f"Some error in getting info from ETSY API: {e}") - pprint.pprint(e) - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.ETSY_API_ERROR, - ) + orders_data = order_cl.get_orders_with_items(page=page_orders) + if orders_data is None: shop_error = True break - - - - - # Get order details and split for creating and updating - for shop_order in shop_orders: - - order, goods_in_order, day, month, client, city = format_order_data( - order=shop_order, - ) - - uploading_orders.orders_data.append( - OrderData( - order=order, - client=client, - city=city, - order_items=goods_in_order, - ) + #TODO подготовить upload + uploading_orders.orders_data.append( + OrderData( + order=order, + client=client, + city=city, + order_items=goods_in_order, ) + ) # res = upload_orders_data(uploading_orders) with open("test_data.json", "w") as f: diff --git a/src/utils/format_datetime.py b/src/utils/format_datetime.py index 5e2035a..1199c59 100644 --- a/src/utils/format_datetime.py +++ b/src/utils/format_datetime.py @@ -8,3 +8,10 @@ def is_iso_utc_z_format(date_str): return dt.utcoffset().total_seconds() == 0 except ValueError: return False + + +def iso_to_simple(iso_str: str): + """iso8601 to dd.mm.YYYY""" + dt = datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return f"{dt.day:02d}.{dt.month:02d}.{dt.year}" + diff --git a/src/utils/format_order_data.py b/src/utils/format_order_data.py index 6d5bee4..7e5f8c6 100644 --- a/src/utils/format_order_data.py +++ b/src/utils/format_order_data.py @@ -1,116 +1,75 @@ import json from datetime import datetime -from schemes.city import City -from schemes.client import Client from schemes.order import Order +from schemes.client import Client +from schemes.city import City from schemes.order_item import GoodInOrder +from schemes.upload_order import OrderData +from utils.format_datetime import iso_to_simple + + +def _format_order(*, + order: dict, order_obj: Order): + """earliest fill order_obj""" + order_obj.buyer_paid = None + order_obj.order_id = order["AmazonOrderId"] + order_obj.status = order["OrderStatus"] # TODO может отличаться от бэка + order_obj.date = iso_to_simple(order["PurchaseDate"]) # formated_date = f"{day}.{month}.{year}" + order_obj.quantity = 0 + order_obj.tax = 0 + + +def _format_good_in_order(*, + item: dict, + item_obj: GoodInOrder): + """fill good_in_order obj""" + item_obj.uniquename = item["SellerSKU"] + item_obj.quantity = item["QuantityOrdered"] + item_obj.amount = (item["ItemPrice"]["Amount"] * item["QuantityOrdered"]) - item["PromotionDiscount"]["Amount"] + item_obj.engraving_info = item["Title"] # чё надо? + +def _format_client(*, + order: dict, + client_obj: Client): + # TODO Другого не дано :( + client_obj.email = order["BuyerInfo"]["BuyerEmail"] -def format_order_data( - order: dict, -): - order_id = order["receipt_id"] - # Good in orders + +def _format_city(*, + order: dict, + city_obj: City): + city_obj.name = order["ShippingAddress"]["City"] + city_obj.state = order["ShippingAddress"]["StateOrRegion"] + city_obj.country = order["ShippingAddress"]["CountryCode"] + + +def format_order_data(*, + order: dict, + items: list[dict]) -> OrderData: + order_obj = Order() + client_obj = Client() + city_obj = City() order_items = [] - order_created_at = datetime.fromtimestamp(order["created_timestamp"]) - ########### - day = order_created_at.day - month = order_created_at.month - year = order_created_at.year - ########### - order_status = order["status"] - # Order date - formated_date = f"{day}.{month}.{year}" - # Full quantity of items in order - full_items_quantity = 0 - - # Getting order shipping info - _shipping_info = order["shipments"] - receipt_shipping_id = "" - tracking_code = "" - if len(_shipping_info): - receipt_shipping_id = str(_shipping_info[0]["receipt_shipping_id"]) - tracking_code = str(_shipping_info[0]["tracking_code"]) - - # Getting order city and state ordered from - city = City() - try: - city = City( - name=order["city"], - state=order["state"], - country=order["country_iso"], - ) - except Exception: - pass - # Getting client info - client = Client() - try: - client = Client( - user_marketplace_id=str(order["buyer_user_id"]), - name=order["name"], - email=order["buyer_email"] - ) - except Exception: - pass - # Creating goods and good in order objects - for trans in order["transactions"]: - # Quantity of item - quantity = trans["quantity"] - # Full quantity of order items - full_items_quantity += quantity - # Name of good - uniquename: str = trans["sku"] - # Transaction ID - listing_id: int = trans.get("listing_id") - # Product ID - product_id: int = trans.get("product_id") - # Transaction Type - transaction_type: str = trans.get("transaction_type") - - # Getting all additional engraving info - engraving_info: dict = {} - for variation in trans["variations"]: - variation_name = variation.get("formatted_name") - variation_value = variation.get("formatted_value") - - engraving_info_item: dict = { - f"{variation_name}": f"{variation_value}", - "listing_id": listing_id, - "product_id": product_id, - "transaction_type": transaction_type, - } - engraving_info.update(engraving_info_item) - - # Convert obj to str - engraving_info_str = json.dumps(engraving_info) - - # Amount of item - price = (trans["price"]["amount"] / trans["price"]["divisor"]) * quantity - amount = price - trans["buyer_coupon"] - ################# - order_items.append( - GoodInOrder( - uniquename=uniquename, - quantity=quantity, - amount=amount, - engraving_info=engraving_info_str, - ) - ) - - order_total = order["grandtotal"] - buyer_paid = order_total["amount"] / order_total["divisor"] - tax_total = order["total_tax_cost"] - tax_amount = tax_total["amount"] / tax_total["divisor"] - order = Order( - status=order_status, - order_id=str(order_id), - date=formated_date, - quantity=full_items_quantity, - buyer_paid=buyer_paid, - tax=tax_amount, - receipt_shipping_id=receipt_shipping_id, - tracking_code=tracking_code, - ) - return order, order_items, day, month, client, city + _format_order(order=order, order_obj=order_obj) + _format_client(order=order, client_obj=client_obj) + _format_city(order=order, city_obj=city_obj) + + for item in items: + good_in_order = GoodInOrder() + _format_good_in_order(item=item, item_obj=good_in_order) + + # calculate tax and quantity for order + order_obj.quantity += good_in_order.quantity + order_obj.tax += (good_in_order.quantity * item["ItemTax"]["Amount"]) - item["PromotionDiscountTax"]["Amount"] + + order_items.append(good_in_order) + + return OrderData( + order=order_obj, + client=client_obj, + city=city_obj, + order_items=order_items + ) From 577ceded74833f6834b1e57d6b9e3e39be782d91 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Thu, 24 Apr 2025 18:42:13 +0300 Subject: [PATCH 08/17] control refactor --- src/parser_debug.py | 66 ++++++--------------------------------------- 1 file changed, 8 insertions(+), 58 deletions(-) diff --git a/src/parser_debug.py b/src/parser_debug.py index 1561a66..a802c92 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -1,51 +1,33 @@ -import concurrent.futures import json -import pprint -from datetime import datetime, timedelta +from datetime import datetime from api.parser import update_parser_status_by_id from constants.status import ParserStatus -from etsy_api.orders import get_all_orders_by_shop_id -from schemes.upload_order import UploadingOrderData, OrderData -from utils.format_order_data import format_order_data +from schemes.shop_data import ShopData +from schemes.upload_order import UploadingOrderData from utils.parser_shops_data import get_parser_shops_data -from utils.excel.write_each_row import write_to_excel -from configs import settings from amazon_api.get_amazon_api import OrderClient - - -from sp_api.util import throttle_retry, load_all_pages - -from configs import settings -from constants.amazon_credentials import CREDENTIALS_ARG from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE -from utils.safe_ratelimit_amazon import safe_rate_limit -from utils.format_datetime import is_iso_utc_z_format from log.logger import logger - # Every 30 minutes PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 -CREATED_AFTER = datetime(2024, 1, 1).isoformat().replace("+00:00", 'Z') EXCEL_FILE = "data/check_point.xlsx" #temporaty #TODO link on update refresh token don't forget!!!! -def process_single_shop(shop): +def process_single_shop(shop: ShopData): - # Initializing constants + order_cl = OrderClient(shop=shop) + created_after = LAST_MONTH_DATE shop_error = False + offset = 0 start_time_shop = datetime.now() - order_cl = OrderClient(log=logger) - offset = 0 - created_after = EARLIEST_DATE - # weekday = datetime.now().weekday() - ########################## logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") logger.info( @@ -59,11 +41,8 @@ def process_single_shop(shop): logger.success(f"Parser status updated.") - uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) - - #loop for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): """Every 100 orders after """ @@ -77,27 +56,14 @@ def process_single_shop(shop): shop_error = True break - #TODO подготовить upload - uploading_orders.orders_data.append( - OrderData( - order=order, - client=client, - city=city, - order_items=goods_in_order, - ) - ) + uploading_orders.orders_data.extend(orders_data) - # res = upload_orders_data(uploading_orders) with open("test_data.json", "w") as f: json.dump(uploading_orders.model_dump(), f) offset += 100 - if offset > 200: - if now_hour == 20 and weekday in (5, 6): - continue - break if shop_error: logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") @@ -120,22 +86,6 @@ def process_single_shop(shop): ) -def etsy_api_parser(): - shops_data = get_parser_shops_data() - - # Используем ThreadPoolExecutor для параллельной обработки - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Запускаем обработку каждого магазина в отдельном потоке - futures = [executor.submit(process_single_shop, shop) for shop in shops_data] - # Ждем завершения всех задач - concurrent.futures.wait(futures) - # Проверяем, были ли исключения - for future in futures: - if future.exception(): - logger.error(f"Error in thread: {future.exception()}") - logger.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") - - if __name__ == "__main__": shops_data = get_parser_shops_data() process_single_shop(shops_data[2]) From 19d74cf63900b2c1d289120f09409c69aafd3f24 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Fri, 25 Apr 2025 01:31:31 +0300 Subject: [PATCH 09/17] complete debug parser --- src/amazon_api/get_amazon_api.py | 4 +--- src/api/parser.py | 19 +++++++++++-------- src/configs/env.py | 15 +-------------- src/constants/files_paths.py | 4 ++-- src/parser_debug.py | 9 ++------- src/utils/format_order_data.py | 23 +++++++++++++++++------ src/utils/safe_ratelimit_amazon.py | 3 --- 7 files changed, 34 insertions(+), 43 deletions(-) diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py index 1731c4e..2db5c1f 100644 --- a/src/amazon_api/get_amazon_api.py +++ b/src/amazon_api/get_amazon_api.py @@ -42,9 +42,6 @@ def _load_all_items(self, order_id, **kwargs): return self.order_api.get_order_items(order_id=order_id, **kwargs) -#TODO format orders and items - - def get_orders_with_items(self, page: ApiResponse) -> list[OrderData] | None: try: for order in page.payload.get('Orders'): @@ -62,4 +59,5 @@ def get_orders_with_items(self, page: ApiResponse) -> list[OrderData] | None: status=ParserStatus.OK_AND_WAIT ) return None + return self._list_orders_data diff --git a/src/api/parser.py b/src/api/parser.py index 642aa87..88b8a60 100644 --- a/src/api/parser.py +++ b/src/api/parser.py @@ -20,15 +20,18 @@ def update_parser_status_by_id( response = req.put( f"{settings.API_URL}/parser/", headers=authorization().model_dump(), json=data ) - except Exception as e: - return update_parser_status_by_id(parser_id, status) - if response.status_code != 200: + if response.status_code != 200: + raise ValueError(f"Unexpected status code: {response.status_code}") + + except ValueError: logger.error( f""" - Some error when updating parser status. - Parser ID: {parser_id} - Status code: {response.status_code} - Details: {response.json()['detail']} - """ + Some error when updating parser status. + Parser ID: {parser_id} + Status code: {response.status_code} + Details: {response.json()['detail']} + """ ) + except Exception as e: + logger.error(f"Some wrong with parser status updating = {e}") diff --git a/src/configs/env.py b/src/configs/env.py index cc2479a..dcfbf5d 100644 --- a/src/configs/env.py +++ b/src/configs/env.py @@ -1,17 +1,3 @@ -# import os - -# from dotenv import load_dotenv - -# from pydantic.v1 import BaseSettings -# -# load_dotenv() -# -# LWA_APP_ID = os.getenv("LWA_APP_ID") -# LWA_CLIENT_SECRET = os.getenv("LWA_CLIENT_SECRET") -# SP_API_REFRESH_TOKEN = os.getenv("SP_API_REFRESH_TOKEN") -# - - from pydantic_settings import BaseSettings, SettingsConfigDict @@ -26,5 +12,6 @@ class Settings(BaseSettings): API_AUTH_TOKEN: str LOG_FILE: str + DATA_FOLDER_PATH: str diff --git a/src/constants/files_paths.py b/src/constants/files_paths.py index 67dd717..a325950 100644 --- a/src/constants/files_paths.py +++ b/src/constants/files_paths.py @@ -1,5 +1,5 @@ -from configs.env import DATA_FOLDER_PATH +from configs import settings -SHOPS_DATA_FILE_PATH = f"{DATA_FOLDER_PATH}/shops/shops.json" +SHOPS_DATA_FILE_PATH = f"{settings.DATA_FOLDER_PATH}/shops/shops.json" diff --git a/src/parser_debug.py b/src/parser_debug.py index a802c92..d7d0760 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -12,10 +12,7 @@ from log.logger import logger -# Every 30 minutes -PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 - -EXCEL_FILE = "data/check_point.xlsx" #temporaty +EXCEL_FILE = "data/check_point.xlsx" #temporary #TODO link on update refresh token don't forget!!!! @@ -50,7 +47,6 @@ def process_single_shop(shop: ShopData): f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." ) - orders_data = order_cl.get_orders_with_items(page=page_orders) if orders_data is None: shop_error = True @@ -61,7 +57,6 @@ def process_single_shop(shop: ShopData): with open("test_data.json", "w") as f: json.dump(uploading_orders.model_dump(), f) - offset += 100 @@ -88,4 +83,4 @@ def process_single_shop(shop: ShopData): if __name__ == "__main__": shops_data = get_parser_shops_data() - process_single_shop(shops_data[2]) + process_single_shop(shops_data[0]) # TODO пофиксить хуйню с json и магазинами diff --git a/src/utils/format_order_data.py b/src/utils/format_order_data.py index 7e5f8c6..599a925 100644 --- a/src/utils/format_order_data.py +++ b/src/utils/format_order_data.py @@ -26,7 +26,10 @@ def _format_good_in_order(*, """fill good_in_order obj""" item_obj.uniquename = item["SellerSKU"] item_obj.quantity = item["QuantityOrdered"] - item_obj.amount = (item["ItemPrice"]["Amount"] * item["QuantityOrdered"]) - item["PromotionDiscount"]["Amount"] + if item.get("ItemPrice") and item.get("PromotionDiscount"): + item_obj.amount = ( + (float(item["ItemPrice"]["Amount"]) * item["QuantityOrdered"]) - float(item["PromotionDiscount"]["Amount"]) + ) item_obj.engraving_info = item["Title"] # чё надо? @@ -34,15 +37,19 @@ def _format_client(*, order: dict, client_obj: Client): # TODO Другого не дано :( - client_obj.email = order["BuyerInfo"]["BuyerEmail"] + buyer_info = order.get("BuyerInfo") + if buyer_info: + client_obj.email = buyer_info["BuyerEmail"] def _format_city(*, order: dict, city_obj: City): - city_obj.name = order["ShippingAddress"]["City"] - city_obj.state = order["ShippingAddress"]["StateOrRegion"] - city_obj.country = order["ShippingAddress"]["CountryCode"] + shipping_address = order.get("ShippingAddress") + if shipping_address: + city_obj.name = shipping_address["City"] + city_obj.state = shipping_address["StateOrRegion"] + city_obj.country = shipping_address["CountryCode"] def format_order_data(*, @@ -63,7 +70,11 @@ def format_order_data(*, # calculate tax and quantity for order order_obj.quantity += good_in_order.quantity - order_obj.tax += (good_in_order.quantity * item["ItemTax"]["Amount"]) - item["PromotionDiscountTax"]["Amount"] + + if item.get("ItemTax") and item.get("PromotionDiscountTax"): + order_obj.tax += ( + (good_in_order.quantity * float(item["ItemTax"]["Amount"])) - float(item["PromotionDiscountTax"]["Amount"]) + ) order_items.append(good_in_order) diff --git a/src/utils/safe_ratelimit_amazon.py b/src/utils/safe_ratelimit_amazon.py index 81bcfa8..2404cdf 100644 --- a/src/utils/safe_ratelimit_amazon.py +++ b/src/utils/safe_ratelimit_amazon.py @@ -12,9 +12,6 @@ def _delay_execution(*, return 1 / float(rate_limit) # for dynamically rate limit return float(throttle_by_seconds) -def _check_next_page(): - - def safe_rate_limit(throttle_by_seconds: int = 1, header_limit: bool = False): From 3edb8d3512b4fe1502aacadc77ac4c06e506f4eb Mon Sep 17 00:00:00 2001 From: Trydimas Date: Fri, 25 Apr 2025 03:23:12 +0300 Subject: [PATCH 10/17] refactor parser_debug.py create parser all parser --- src/api/order.py | 21 +++--- src/parser_2.py | 123 +++++++++++++++++++++++++++++++++ src/parser_all_2.py | 115 ++++++++++++++++++++++++++++++ src/parser_debug.py | 9 +-- src/utils/format_order_data.py | 1 - src/utils/retry.py | 22 ++++++ 6 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 src/parser_2.py create mode 100644 src/parser_all_2.py create mode 100644 src/utils/retry.py diff --git a/src/api/order.py b/src/api/order.py index 159db23..b5b7660 100644 --- a/src/api/order.py +++ b/src/api/order.py @@ -1,23 +1,28 @@ import requests as req -from loguru import logger as log from api.auth import authorization -from configs.env import API_URL +from configs import settings from schemes.upload_order import UploadingOrderData +from log.logger import logger +from utils.retry import retry -def upload_orders_data(orders: UploadingOrderData) -> bool: + +@retry() +def upload_orders_data(orders: UploadingOrderData): response = req.post( - f"{API_URL}/parser/orders/upload/", + f"{settings.API_URL}/parser/orders/upload/", headers=authorization().model_dump(), json=orders.model_dump(), ) - if response.status_code != 200: - log.error( + if response.status_code != 200: # TODO поиграться с проверкой + logger.error( f""" Some error when uploading orders data, status code: {response.status_code} """ ) - return False - return True + response.raise_for_status() + + + diff --git a/src/parser_2.py b/src/parser_2.py new file mode 100644 index 0000000..68793c0 --- /dev/null +++ b/src/parser_2.py @@ -0,0 +1,123 @@ +import json +from datetime import datetime + +from api.parser import update_parser_status_by_id +from api.order import upload_orders_data +from constants.status import ParserStatus +from schemes.shop_data import ShopData +from schemes.upload_order import UploadingOrderData +from utils.parser_shops_data import get_parser_shops_data +from amazon_api.get_amazon_api import OrderClient + +from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE +from log.logger import logger + +EXCEL_FILE = "data/check_point.xlsx" # temporary +RETRY_LIMIT = 10 + + +# TODO link on update refresh token don't forget!!!! + +def process_single_shop(shop: ShopData): + order_cl = OrderClient(shop=shop) + created_after = LAST_MONTH_DATE + shop_error = False + offset = 0 + start_time_shop = datetime.now() + weekday = datetime.now().weekday() + now_hour = datetime.now().hour + + logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") + logger.info( + f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." + ) + + update_parser_status_by_id( + parser_id=shop.parser_id, + status=ParserStatus.PARSING, + ) + + logger.success(f"Parser status updated.") + + for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): + """Every 100 orders after """ + + uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) + + logger.info( + f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." + ) + + orders_data = order_cl.get_orders_with_items(page=page_orders) + if orders_data is None: + shop_error = True + break + + uploading_orders.orders_data.extend(orders_data) + + try: + upload_orders_data(uploading_orders) + except: + logger.critical(f"Some error on sending info to backend") + update_parser_status_by_id( + parser_id=shop.parser_id, + status=ParserStatus.ETSY_API_ERROR, + ) + shop_error = True + + offset += 100 + + if offset > 200: + if now_hour == 20 and weekday in (5, 6): + continue + break + + if shop_error: + logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") + return + logger.info( + f"Updating parser {shop.parser_id} status to {ParserStatus.OK_AND_WAIT}..." + ) + + update_parser_status_by_id( + parser_id=shop.parser_id, + status=ParserStatus.OK_AND_WAIT, + last_parsed=datetime.now().timestamp(), + ) + + logger.success(f"Parser status updated.") + logger.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") + end_time_shop = datetime.now() + logger.info( + f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" + ) + +#TODO func +def etsy_api_parser(): + shops_data = get_parser_shops_data() + # for shop in shops_data: + # etsy_api = get_etsy_api(shop.shop_id) + # Используем ThreadPoolExecutor для параллельной обработки + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + # Запускаем обработку каждого магазина в отдельном потоке + futures = [executor.submit(process_single_shop, shop) for shop in shops_data] + # Ждем завершения всех задач + concurrent.futures.wait(futures) + # Проверяем, были ли исключения + for future in futures: + if future.exception(): + log.error(f"Error in thread: {future.exception()}") + log.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") + + +# TODO: сделать чтобы файлы с кредами не писались в файл в многопотоке +#TODO Func + +if __name__ == "__main__": + while True: + try: + etsy_api_parser() + time.sleep(PARSER_WAIT_TIME_IN_SECONDS) + except Exception as e: + log.error(f"Error on fetching orders {e}") + time.sleep(900) diff --git a/src/parser_all_2.py b/src/parser_all_2.py new file mode 100644 index 0000000..4c9573c --- /dev/null +++ b/src/parser_all_2.py @@ -0,0 +1,115 @@ +import json +from datetime import datetime + +from api.parser import update_parser_status_by_id +from api.order import upload_orders_data +from constants.status import ParserStatus +from schemes.shop_data import ShopData +from schemes.upload_order import UploadingOrderData +from utils.parser_shops_data import get_parser_shops_data +from amazon_api.get_amazon_api import OrderClient + +from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE +from log.logger import logger + +EXCEL_FILE = "data/check_point.xlsx" # temporary +RETRY_LIMIT = 10 + + +# TODO link on update refresh token don't forget!!!! + +def process_single_shop(shop: ShopData): + order_cl = OrderClient(shop=shop) + created_after = EARLIEST_DATE + shop_error = False + offset = 0 + start_time_shop = datetime.now() + + logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") + logger.info( + f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." + ) + + update_parser_status_by_id( + parser_id=shop.parser_id, + status=ParserStatus.PARSING, + ) + + logger.success(f"Parser status updated.") + + for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): + """Every 100 orders after """ + + uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) + + logger.info( + f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." + ) + + orders_data = order_cl.get_orders_with_items(page=page_orders) + if orders_data is None: + shop_error = True + break + + uploading_orders.orders_data.extend(orders_data) + + try: + upload_orders_data(uploading_orders) + except: + logger.critical(f"Some error on sending info to backend") + update_parser_status_by_id( + parser_id=shop.parser_id, + status=ParserStatus.ETSY_API_ERROR, + ) + shop_error = True + + offset += 100 + + if shop_error: + logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") + return + logger.info( + f"Updating parser {shop.parser_id} status to {ParserStatus.OK_AND_WAIT}..." + ) + + update_parser_status_by_id( + parser_id=shop.parser_id, + status=ParserStatus.OK_AND_WAIT, + last_parsed=datetime.now().timestamp(), + ) + + logger.success(f"Parser status updated.") + logger.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") + end_time_shop = datetime.now() + logger.info( + f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" + ) + + +#TODO this func ref +def etsy_api_parser(): + shops_data = get_parser_shops_data() + + # Используем ThreadPoolExecutor для параллельной обработки + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + # Запускаем обработку каждого магазина в отдельном потоке + futures = [executor.submit(process_single_shop, shop) for shop in shops_data] + # Ждем завершения всех задач + concurrent.futures.wait(futures) + # Проверяем, были ли исключения + for future in futures: + if future.exception(): + log.error(f"Error in thread: {future.exception()}") + log.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") + + +#TODO this func ref +if __name__ == "__main__": + + try: + etsy_api_parser() + time.sleep(PARSER_WAIT_TIME_IN_SECONDS) + time.sleep(900) + except Exception as e: + log.error(f"Error on fetching orders {e}") + time.sleep(900) diff --git a/src/parser_debug.py b/src/parser_debug.py index d7d0760..b6417b3 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -12,11 +12,6 @@ from log.logger import logger -EXCEL_FILE = "data/check_point.xlsx" #temporary - - -#TODO link on update refresh token don't forget!!!! - def process_single_shop(shop: ShopData): order_cl = OrderClient(shop=shop) @@ -38,11 +33,11 @@ def process_single_shop(shop: ShopData): logger.success(f"Parser status updated.") - uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) - for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): """Every 100 orders after """ + uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) + logger.info( f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." ) diff --git a/src/utils/format_order_data.py b/src/utils/format_order_data.py index 599a925..5526757 100644 --- a/src/utils/format_order_data.py +++ b/src/utils/format_order_data.py @@ -36,7 +36,6 @@ def _format_good_in_order(*, def _format_client(*, order: dict, client_obj: Client): - # TODO Другого не дано :( buyer_info = order.get("BuyerInfo") if buyer_info: client_obj.email = buyer_info["BuyerEmail"] diff --git a/src/utils/retry.py b/src/utils/retry.py new file mode 100644 index 0000000..e37e3ee --- /dev/null +++ b/src/utils/retry.py @@ -0,0 +1,22 @@ +from pprint import pprint + +def retry(retry=10, exception_classes=None): + if exception_classes is None: + exception_classes = (Exception,) + + def decorator(function): + def wrapper(*args, **kwargs): + attempts = 0 + while attempts < retry: + try: + return function(*args, **kwargs) + except exception_classes as e: + attempts += 1 + if attempts >= retry: + raise e + pprint(f"Retrying {function.__name__} (Attempt {attempts}/{retry}) due to error: {e}") + + wrapper.__doc__ = function.__doc__ + return wrapper + + return decorator From 5c3e601e95da81599c8f0cadd0cfdcaf6018364b Mon Sep 17 00:00:00 2001 From: Trydimas Date: Fri, 25 Apr 2025 16:07:37 +0300 Subject: [PATCH 11/17] refactor parser all2 parser 2 --- src/amazon_api/get_amazon_api.py | 1 + src/constants/amazon_dates.py | 5 ++++- src/constants/files_paths.py | 2 +- src/parser_2.py | 33 +++++++++++++----------------- src/parser_all_2.py | 32 +++++++++-------------------- src/utils/excel/write_each_row.py | 21 ------------------- src/utils/retry.py | 4 ++-- src/utils/safe_ratelimit_amazon.py | 3 --- 8 files changed, 32 insertions(+), 69 deletions(-) delete mode 100644 src/utils/excel/write_each_row.py diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py index 2db5c1f..4aee8ad 100644 --- a/src/amazon_api/get_amazon_api.py +++ b/src/amazon_api/get_amazon_api.py @@ -46,6 +46,7 @@ def get_orders_with_items(self, page: ApiResponse) -> list[OrderData] | None: try: for order in page.payload.get('Orders'): _order_id = order["AmazonOrderId"] + pprint.pprint(f"formating order ID: {_order_id}") order_data = format_order_data( order=order, items=self._get_all_items(order_id=_order_id) diff --git a/src/constants/amazon_dates.py b/src/constants/amazon_dates.py index 41d7e61..a255d45 100644 --- a/src/constants/amazon_dates.py +++ b/src/constants/amazon_dates.py @@ -9,4 +9,7 @@ CURRENT_DATE=datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') #last month date -LAST_MONTH_DATE = (datetime.now(timezone.utc) - timedelta(30)).isoformat().replace("+00:00", "Z") \ No newline at end of file +LAST_MONTH_DATE = (datetime.now(timezone.utc) - timedelta(30)).isoformat().replace("+00:00", "Z") + +#last week date +LAST_WEEK_DATE = (datetime.now(timezone.utc) - timedelta(7)).isoformat().replace("+00:00", "Z") \ No newline at end of file diff --git a/src/constants/files_paths.py b/src/constants/files_paths.py index a325950..7c866fe 100644 --- a/src/constants/files_paths.py +++ b/src/constants/files_paths.py @@ -1,5 +1,5 @@ from configs import settings -SHOPS_DATA_FILE_PATH = f"{settings.DATA_FOLDER_PATH}/shops/shops.json" +SHOPS_DATA_FILE_PATH = f"{settings.DATA_FOLDER_PATH}/shops/shops_amazon.json" diff --git a/src/parser_2.py b/src/parser_2.py index 68793c0..ca6a45c 100644 --- a/src/parser_2.py +++ b/src/parser_2.py @@ -1,5 +1,7 @@ +import concurrent.futures import json -from datetime import datetime +import time +from datetime import datetime, timezone from api.parser import update_parser_status_by_id from api.order import upload_orders_data @@ -12,12 +14,9 @@ from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE from log.logger import logger -EXCEL_FILE = "data/check_point.xlsx" # temporary -RETRY_LIMIT = 10 +PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 -# TODO link on update refresh token don't forget!!!! - def process_single_shop(shop: ShopData): order_cl = OrderClient(shop=shop) created_after = LAST_MONTH_DATE @@ -56,7 +55,7 @@ def process_single_shop(shop: ShopData): uploading_orders.orders_data.extend(orders_data) try: - upload_orders_data(uploading_orders) + upload_orders_data(uploading_orders) # upload data to backend except: logger.critical(f"Some error on sending info to backend") update_parser_status_by_id( @@ -92,26 +91,22 @@ def process_single_shop(shop: ShopData): f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" ) -#TODO func + def etsy_api_parser(): shops_data = get_parser_shops_data() - # for shop in shops_data: - # etsy_api = get_etsy_api(shop.shop_id) - # Используем ThreadPoolExecutor для параллельной обработки + + # using ThreadPoolExecutor for parallel processing with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Запускаем обработку каждого магазина в отдельном потоке + # Starting process for each shop in a separate thread futures = [executor.submit(process_single_shop, shop) for shop in shops_data] - # Ждем завершения всех задач + # Waiting for all tasks to complete concurrent.futures.wait(futures) - # Проверяем, были ли исключения + # checking for any exceptions for future in futures: if future.exception(): - log.error(f"Error in thread: {future.exception()}") - log.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") - + logger.error(f"Error in thread: {future.exception()}") + logger.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") -# TODO: сделать чтобы файлы с кредами не писались в файл в многопотоке -#TODO Func if __name__ == "__main__": while True: @@ -119,5 +114,5 @@ def etsy_api_parser(): etsy_api_parser() time.sleep(PARSER_WAIT_TIME_IN_SECONDS) except Exception as e: - log.error(f"Error on fetching orders {e}") + logger.error(f"Error on fetching orders {e}") time.sleep(900) diff --git a/src/parser_all_2.py b/src/parser_all_2.py index 4c9573c..293e554 100644 --- a/src/parser_all_2.py +++ b/src/parser_all_2.py @@ -1,4 +1,4 @@ -import json +import concurrent.futures from datetime import datetime from api.parser import update_parser_status_by_id @@ -9,18 +9,13 @@ from utils.parser_shops_data import get_parser_shops_data from amazon_api.get_amazon_api import OrderClient -from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE +from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE, LAST_WEEK_DATE from log.logger import logger -EXCEL_FILE = "data/check_point.xlsx" # temporary -RETRY_LIMIT = 10 - - -# TODO link on update refresh token don't forget!!!! def process_single_shop(shop: ShopData): order_cl = OrderClient(shop=shop) - created_after = EARLIEST_DATE + created_after = LAST_WEEK_DATE shop_error = False offset = 0 start_time_shop = datetime.now() @@ -54,7 +49,7 @@ def process_single_shop(shop: ShopData): uploading_orders.orders_data.extend(orders_data) try: - upload_orders_data(uploading_orders) + upload_orders_data(uploading_orders) # send data to backend except: logger.critical(f"Some error on sending info to backend") update_parser_status_by_id( @@ -86,30 +81,23 @@ def process_single_shop(shop: ShopData): ) -#TODO this func ref def etsy_api_parser(): shops_data = get_parser_shops_data() - # Используем ThreadPoolExecutor для параллельной обработки + # using ThreadPoolExecutor for parallel processing with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Запускаем обработку каждого магазина в отдельном потоке + # Starting process for each shop in a separate thread futures = [executor.submit(process_single_shop, shop) for shop in shops_data] - # Ждем завершения всех задач + # Waiting for all tasks to complete concurrent.futures.wait(futures) - # Проверяем, были ли исключения + # checking for any exceptions for future in futures: if future.exception(): - log.error(f"Error in thread: {future.exception()}") - log.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") + logger.error(f"Error in thread: {future.exception()}") -#TODO this func ref if __name__ == "__main__": - try: etsy_api_parser() - time.sleep(PARSER_WAIT_TIME_IN_SECONDS) - time.sleep(900) except Exception as e: - log.error(f"Error on fetching orders {e}") - time.sleep(900) + logger.error(f"Error on fetching orders {e}") diff --git a/src/utils/excel/write_each_row.py b/src/utils/excel/write_each_row.py deleted file mode 100644 index 08ad44e..0000000 --- a/src/utils/excel/write_each_row.py +++ /dev/null @@ -1,21 +0,0 @@ -import openpyxl -from openpyxl import load_workbook -import os - - -def write_to_excel(value, filename='output.xlsx'): - if os.path.exists(filename): - wb = load_workbook(filename) - else: - wb = openpyxl.Workbook() - - sheet = wb.active - - #finde first empty row in cell A - next_row = sheet.max_row + 1 - while sheet.cell(row=next_row, column=1).value is not None: - next_row += 1 - - sheet.cell(row=next_row, column=1).value = value - - wb.save(filename) \ No newline at end of file diff --git a/src/utils/retry.py b/src/utils/retry.py index e37e3ee..6691e31 100644 --- a/src/utils/retry.py +++ b/src/utils/retry.py @@ -1,4 +1,4 @@ -from pprint import pprint +from log.logger import logger def retry(retry=10, exception_classes=None): if exception_classes is None: @@ -14,7 +14,7 @@ def wrapper(*args, **kwargs): attempts += 1 if attempts >= retry: raise e - pprint(f"Retrying {function.__name__} (Attempt {attempts}/{retry}) due to error: {e}") + logger.critical(f"Retrying {function.__name__} (Attempt {attempts}/{retry}) due to error: {e}") wrapper.__doc__ = function.__doc__ return wrapper diff --git a/src/utils/safe_ratelimit_amazon.py b/src/utils/safe_ratelimit_amazon.py index 2404cdf..50b64ca 100644 --- a/src/utils/safe_ratelimit_amazon.py +++ b/src/utils/safe_ratelimit_amazon.py @@ -23,15 +23,12 @@ def wrapper(*args, **kwargs): if resp.next_token: #excludes delay for several pages return resp - logger.info(resp) - logger.info(f"в декораторе и прошёл первичную проверку....") sleep_time = _delay_execution(throttle_by_seconds=throttle_by_seconds, header_limit=header_limit, rate_limit=resp.rate_limit) if sleep_time: - logger.info(f"запуск слипа....{sleep_time}") time.sleep(sleep_time) return resp From 19a3e7929b70844448a3b450f05149527529aec8 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Fri, 25 Apr 2025 16:39:40 +0300 Subject: [PATCH 12/17] clear unnecessary file --- src/amazon_api/orders.py | 11 --- src/auth_code_endpoint.py | 19 ----- src/constants/commands.py | 4 - src/constants/etsy_oauth.py | 27 ------ src/constants/files_paths.py | 5 -- src/constants/shops_names.py | 5 -- src/db_handler_main.py | 48 ----------- src/etsy_api/__init__.py | 0 src/etsy_api/get_etsy_api.py | 154 ----------------------------------- src/etsy_api/orders.py | 13 --- src/etsy_api/shops.py | 20 ----- src/main_start.py | 10 --- src/parser_2.py | 118 --------------------------- src/parser_all_2.py | 103 ----------------------- src/schemes/access_token.py | 7 -- src/schemes/parser_info.py | 9 -- src/utils/excel/__init__.py | 0 17 files changed, 553 deletions(-) delete mode 100644 src/amazon_api/orders.py delete mode 100644 src/auth_code_endpoint.py delete mode 100644 src/constants/commands.py delete mode 100644 src/constants/etsy_oauth.py delete mode 100644 src/constants/files_paths.py delete mode 100644 src/constants/shops_names.py delete mode 100644 src/db_handler_main.py delete mode 100644 src/etsy_api/__init__.py delete mode 100644 src/etsy_api/get_etsy_api.py delete mode 100644 src/etsy_api/orders.py delete mode 100644 src/etsy_api/shops.py delete mode 100644 src/main_start.py delete mode 100644 src/parser_2.py delete mode 100644 src/parser_all_2.py delete mode 100644 src/schemes/access_token.py delete mode 100644 src/schemes/parser_info.py delete mode 100644 src/utils/excel/__init__.py diff --git a/src/amazon_api/orders.py b/src/amazon_api/orders.py deleted file mode 100644 index 0ef14be..0000000 --- a/src/amazon_api/orders.py +++ /dev/null @@ -1,11 +0,0 @@ -from amazon_api.get_amazon_api import OrderClient - - - -def get_all_orders(): - order_client = OrderClient() - orders = order_client.get_orders_with_items() # type: ignore - - return orders - - diff --git a/src/auth_code_endpoint.py b/src/auth_code_endpoint.py deleted file mode 100644 index aa888d2..0000000 --- a/src/auth_code_endpoint.py +++ /dev/null @@ -1,19 +0,0 @@ -import json - -import uvicorn -from constants.auth_code import AUTH_CODE_RESPONSE_FILE_PATH -from fastapi import FastAPI, Response - -app = FastAPI() - - -@app.get("/auth_code") -def get_auth_code(code: str, state: str): - with open(AUTH_CODE_RESPONSE_FILE_PATH, 'w') as f: - json.dump({"code": code}, f) - - return Response(status_code=200) - - -if __name__ == '__main__': - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/src/constants/commands.py b/src/constants/commands.py deleted file mode 100644 index c07450c..0000000 --- a/src/constants/commands.py +++ /dev/null @@ -1,4 +0,0 @@ -class ParserCommand: - NO_COMMAND = 0 - PARSE_NOW = 1 - UPDATE_ETSY_COOKIE = 2 diff --git a/src/constants/etsy_oauth.py b/src/constants/etsy_oauth.py deleted file mode 100644 index 3bef3a7..0000000 --- a/src/constants/etsy_oauth.py +++ /dev/null @@ -1,27 +0,0 @@ -from etsyv3.util.auth.auth_helper import AuthHelper - -from configs.env import ETSY_API_KEY, ETSY_API_REDIRECT_URL, CODE_VERIFIER - -ETSY_OAUTH_CONNECT_URL = "https://www.etsy.com/oauth/connect" -ETSY_OAUTH_TOKEN_URL = "https://api.etsy.com/v3/public/oauth/token" - -SCOPES = [ - "listings_r", - "transactions_r", - "billing_r", - "shops_r", - "address_r", - "profile_r", - "feedback_r", - "recommend_r", -] - -STATE = "superstate" - -etsy_auth = AuthHelper( - keystring=ETSY_API_KEY, - redirect_uri=ETSY_API_REDIRECT_URL, - scopes=SCOPES, - code_verifier=CODE_VERIFIER, - state=STATE, -) diff --git a/src/constants/files_paths.py b/src/constants/files_paths.py deleted file mode 100644 index 7c866fe..0000000 --- a/src/constants/files_paths.py +++ /dev/null @@ -1,5 +0,0 @@ -from configs import settings - -SHOPS_DATA_FILE_PATH = f"{settings.DATA_FOLDER_PATH}/shops/shops_amazon.json" - - diff --git a/src/constants/shops_names.py b/src/constants/shops_names.py deleted file mode 100644 index 0fdc7dc..0000000 --- a/src/constants/shops_names.py +++ /dev/null @@ -1,5 +0,0 @@ -class ShopName: - NIKO = "NiKoEngraving" - ALDA = "AldaProduction" - DADA = "DaDaTeamEngraving" - ELMA = "ElmaVadaStudio" diff --git a/src/db_handler_main.py b/src/db_handler_main.py deleted file mode 100644 index 4f82395..0000000 --- a/src/db_handler_main.py +++ /dev/null @@ -1,48 +0,0 @@ -import json -import time - -from api.command_handler import get_parser_info, update_parser_command_to_default -from configs.env import DATA_FOLDER_PATH -from constants.commands import ParserCommand -from schemes.parser_info import Parser - - -def update_commands_json(command: int, parser_id: int): - with open(f"{DATA_FOLDER_PATH}/shops/commands.json", "r") as f: - data = json.load(f) - - data[parser_id - 1]["command"] = command - - with open(f"{DATA_FOLDER_PATH}/shops/commands.json", "w") as f: - json.dump(data, f) - - print(data) - - -def command_check(parser: Parser): - command = parser.command - if command == ParserCommand.NO_COMMAND: - return - update_commands_json(command, parser.id) - update_parser_command_to_default(parser.id) - # elif command == ParserCommand.PARSE_NOW: - # pass - # elif command == ParserCommand.UPDATE_ETSY_COOKIE: - # pass - - -def main(): - while True: - try: - with open(f"{DATA_FOLDER_PATH}/shops/shops.json", "r") as f: - data = json.load(f) - except Exception: - continue - for shop in data: - parser = get_parser_info(shop["parser_id"]) - command_check(parser) - time.sleep(15) - - -if __name__ == "__main__": - main() diff --git a/src/etsy_api/__init__.py b/src/etsy_api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/etsy_api/get_etsy_api.py b/src/etsy_api/get_etsy_api.py deleted file mode 100644 index 3550de0..0000000 --- a/src/etsy_api/get_etsy_api.py +++ /dev/null @@ -1,154 +0,0 @@ -import json -import time -from datetime import datetime, timedelta - -from etsyv3 import EtsyAPI -from loguru import logger as log -from typing_extensions import deprecated - -from configs.env import ETSY_API_KEY -from constants.etsy_oauth import etsy_auth, STATE -from constants.files_paths import SHOPS_DATA_FILE_PATH -from schemes.access_token import AuthToken -from schemes.auth import AuthCode -from schemes.shop_data import ShopData - -AUTH_CODE_WAIT_TIME_IN_SECONDS = 15 -AUTH_TOKEN_LIFE_TIME_IN_SECONDS = 3600 - - -class SouvTechEtsyAPI(EtsyAPI): - shop_id: int - - def refresh(self) -> tuple[str, str, datetime]: - log.info(f"Custom refreshing Etsy access token..") - data = { - "grant_type": "refresh_token", - "client_id": self.keystring, - "refresh_token": self.refresh_token, - } - del self.session.headers["Authorization"] - r = self.session.post("https://api.etsy.com/v3/public/oauth/token", data=data) - log.info(f"Refresh token status code: {r.status_code}") - refreshed = r.json() - log.info(f"Refresh token response: {refreshed}") - self.token = refreshed["access_token"] - self.refresh_token = refreshed["refresh_token"] - tmp_expiry = datetime.utcnow() + timedelta(seconds=refreshed["expires_in"]) - self.expiry = tmp_expiry - self.session.headers["Authorization"] = "Bearer " + self.token - if self.refresh_save is not None: - self.refresh_save(self.token, self.refresh_token, self.expiry) - - # Update access token info in config - log.success(f"New access token: {self.token}") - log.success(f"New refresh token: {self.refresh_token}") - log.success(f"New expiry: {self.expiry}") - - new_auth_token = AuthToken( - access_token=self.token, - refresh_token=self.refresh_token, - expires_at=self.expiry.timestamp(), - ) - _save_auth_token(new_auth_token, self.shop_id) - ######### - - return self.token, self.refresh_token, self.expiry - - -def _get_shop_data_by_id(shop_id: int) -> ShopData: - with open(SHOPS_DATA_FILE_PATH) as f: - shops_data = [ShopData(**shop) for shop in json.load(f)] - for shop in shops_data: - if shop.shop_id == shop_id: - return shop - - -def _save_auth_token(auth_token: AuthToken, shop_id: int): - with open(SHOPS_DATA_FILE_PATH) as f: - shops_data = [ShopData(**shop_data) for shop_data in json.load(f)] - for shop in shops_data: - if shop.shop_id == shop_id: - shop.shop_token = auth_token.access_token - shop.shop_refresh_token = auth_token.refresh_token - shop.expiry = auth_token.expires_at - with open(SHOPS_DATA_FILE_PATH, 'w') as f: - json.dump([shop_data.model_dump() for shop_data in shops_data], f) - - -def _get_auth_code(shop_id: int) -> AuthCode: - shop_data = _get_shop_data_by_id(shop_id) - - auth_code_exists = shop_data.shop_auth_code.strip() - if not auth_code_exists: - auth_url, _ = etsy_auth.get_auth_code() - print(f"Auth code is not found. Open {auth_url} to grant access.") - time.sleep(AUTH_CODE_WAIT_TIME_IN_SECONDS) - return _get_auth_code(shop_id) - - auth_code_response = AuthCode(code=shop_data.shop_auth_code) - return auth_code_response - - -def _get_auth_token(shop_id: int) -> AuthToken: - shop_data = _get_shop_data_by_id(shop_id) - - auth_token_exists = shop_data.shop_token.strip() and shop_data.shop_refresh_token.strip() and shop_data.expiry != 0 - if not auth_token_exists: - auth_code_response = _get_auth_code(shop_id) - etsy_auth.set_authorisation_code( - code=auth_code_response.code, - state=STATE, - ) - auth_token_response = etsy_auth.get_access_token() - log.info(f"Auth token response: {auth_token_response}") - auth_token = AuthToken( - access_token=auth_token_response['access_token'], - refresh_token=auth_token_response['refresh_token'], - expires_at=auth_token_response['expires_at'], - ) - _save_auth_token(auth_token, shop_id) - return _get_auth_token(shop_id) - - auth_token = AuthToken( - access_token=shop_data.shop_token, - refresh_token=shop_data.shop_refresh_token, - expires_at=shop_data.expiry, - ) - return auth_token - - -@deprecated("Token refresh inside EtsyAPI, not needed") -def refresh_auth_token(etsy_api: SouvTechEtsyAPI, shop_id: int): - new_access_token, new_refresh_token, new_expires_at = etsy_api.refresh() - log.success(f"New access token: {new_access_token}") - log.success(f"New refresh token: {new_refresh_token}") - log.success(f"New expiry: {new_expires_at}") - new_auth_token = AuthToken( - access_token=new_access_token, - refresh_token=new_refresh_token, - expires_at=new_expires_at.timestamp(), - ) - _save_auth_token(new_auth_token, shop_id) - - -def get_etsy_api(shop_id: int): - auth_token = _get_auth_token(shop_id) - - etsy_api = SouvTechEtsyAPI( - keystring=ETSY_API_KEY, - token=auth_token.access_token, - refresh_token=auth_token.refresh_token, - expiry=datetime.fromtimestamp(auth_token.expires_at), - ) - etsy_api.shop_id = shop_id - time.sleep(3) - # try: - # etsy_api.ping() - # except Unauthorised: - # log.warning(f"Token is expired. Requesting new token.") - # time.sleep(10) - # refresh_auth_token(etsy_api, shop_id) - # return get_etsy_api(shop_id) - - return etsy_api diff --git a/src/etsy_api/orders.py b/src/etsy_api/orders.py deleted file mode 100644 index 32d76b8..0000000 --- a/src/etsy_api/orders.py +++ /dev/null @@ -1,13 +0,0 @@ -from etsy_api.get_etsy_api import get_etsy_api - - -def get_all_orders_by_shop_id(etsy_shop_id: int, shop_id: int, limit: int = 100, offset: int = 0): - etsy_api = get_etsy_api(shop_id) - orders = etsy_api.get_shop_receipts( - shop_id=etsy_shop_id, - was_paid=None, - was_shipped=None, - limit=limit, - offset=offset, - ) - return orders['results'], orders['count'] diff --git a/src/etsy_api/shops.py b/src/etsy_api/shops.py deleted file mode 100644 index c696933..0000000 --- a/src/etsy_api/shops.py +++ /dev/null @@ -1,20 +0,0 @@ -from constants.shops_names import ShopName -from etsy_api.get_etsy_api import get_etsy_api - - -def find_shop_by_name(shop_name: str, shop_id: int): - etsy_api = get_etsy_api(shop_id) - shop = etsy_api.find_shops(shop_name) - return shop - - -if __name__ == "__main__": - shop_name = ShopName.ALDA - shop = find_shop_by_name(shop_name, shop_id=2) - print(shop) - # with open(f"{shop_name}.json", 'w') as f: - # json.dump(shop, f) - - # shop_id = 50508356 - # shop = get_shop_by_id(shop_id) - # print(shop) diff --git a/src/main_start.py b/src/main_start.py deleted file mode 100644 index 70cc909..0000000 --- a/src/main_start.py +++ /dev/null @@ -1,10 +0,0 @@ -from amazon_api.orders import get_all_orders -import json - - -if __name__ == "__main__": - orders = get_all_orders() - with open("check.json", 'W') as file: - json.dump(orders, file, indent=4) - print("ok") - print(orders) \ No newline at end of file diff --git a/src/parser_2.py b/src/parser_2.py deleted file mode 100644 index ca6a45c..0000000 --- a/src/parser_2.py +++ /dev/null @@ -1,118 +0,0 @@ -import concurrent.futures -import json -import time -from datetime import datetime, timezone - -from api.parser import update_parser_status_by_id -from api.order import upload_orders_data -from constants.status import ParserStatus -from schemes.shop_data import ShopData -from schemes.upload_order import UploadingOrderData -from utils.parser_shops_data import get_parser_shops_data -from amazon_api.get_amazon_api import OrderClient - -from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE -from log.logger import logger - -PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 - - -def process_single_shop(shop: ShopData): - order_cl = OrderClient(shop=shop) - created_after = LAST_MONTH_DATE - shop_error = False - offset = 0 - start_time_shop = datetime.now() - weekday = datetime.now().weekday() - now_hour = datetime.now().hour - - logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") - logger.info( - f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." - ) - - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.PARSING, - ) - - logger.success(f"Parser status updated.") - - for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): - """Every 100 orders after """ - - uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) - - logger.info( - f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." - ) - - orders_data = order_cl.get_orders_with_items(page=page_orders) - if orders_data is None: - shop_error = True - break - - uploading_orders.orders_data.extend(orders_data) - - try: - upload_orders_data(uploading_orders) # upload data to backend - except: - logger.critical(f"Some error on sending info to backend") - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.ETSY_API_ERROR, - ) - shop_error = True - - offset += 100 - - if offset > 200: - if now_hour == 20 and weekday in (5, 6): - continue - break - - if shop_error: - logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") - return - logger.info( - f"Updating parser {shop.parser_id} status to {ParserStatus.OK_AND_WAIT}..." - ) - - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.OK_AND_WAIT, - last_parsed=datetime.now().timestamp(), - ) - - logger.success(f"Parser status updated.") - logger.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") - end_time_shop = datetime.now() - logger.info( - f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" - ) - - -def etsy_api_parser(): - shops_data = get_parser_shops_data() - - # using ThreadPoolExecutor for parallel processing - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Starting process for each shop in a separate thread - futures = [executor.submit(process_single_shop, shop) for shop in shops_data] - # Waiting for all tasks to complete - concurrent.futures.wait(futures) - # checking for any exceptions - for future in futures: - if future.exception(): - logger.error(f"Error in thread: {future.exception()}") - logger.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") - - -if __name__ == "__main__": - while True: - try: - etsy_api_parser() - time.sleep(PARSER_WAIT_TIME_IN_SECONDS) - except Exception as e: - logger.error(f"Error on fetching orders {e}") - time.sleep(900) diff --git a/src/parser_all_2.py b/src/parser_all_2.py deleted file mode 100644 index 293e554..0000000 --- a/src/parser_all_2.py +++ /dev/null @@ -1,103 +0,0 @@ -import concurrent.futures -from datetime import datetime - -from api.parser import update_parser_status_by_id -from api.order import upload_orders_data -from constants.status import ParserStatus -from schemes.shop_data import ShopData -from schemes.upload_order import UploadingOrderData -from utils.parser_shops_data import get_parser_shops_data -from amazon_api.get_amazon_api import OrderClient - -from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE, LAST_WEEK_DATE -from log.logger import logger - - -def process_single_shop(shop: ShopData): - order_cl = OrderClient(shop=shop) - created_after = LAST_WEEK_DATE - shop_error = False - offset = 0 - start_time_shop = datetime.now() - - logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") - logger.info( - f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." - ) - - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.PARSING, - ) - - logger.success(f"Parser status updated.") - - for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): - """Every 100 orders after """ - - uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) - - logger.info( - f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." - ) - - orders_data = order_cl.get_orders_with_items(page=page_orders) - if orders_data is None: - shop_error = True - break - - uploading_orders.orders_data.extend(orders_data) - - try: - upload_orders_data(uploading_orders) # send data to backend - except: - logger.critical(f"Some error on sending info to backend") - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.ETSY_API_ERROR, - ) - shop_error = True - - offset += 100 - - if shop_error: - logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") - return - logger.info( - f"Updating parser {shop.parser_id} status to {ParserStatus.OK_AND_WAIT}..." - ) - - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.OK_AND_WAIT, - last_parsed=datetime.now().timestamp(), - ) - - logger.success(f"Parser status updated.") - logger.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") - end_time_shop = datetime.now() - logger.info( - f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" - ) - - -def etsy_api_parser(): - shops_data = get_parser_shops_data() - - # using ThreadPoolExecutor for parallel processing - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Starting process for each shop in a separate thread - futures = [executor.submit(process_single_shop, shop) for shop in shops_data] - # Waiting for all tasks to complete - concurrent.futures.wait(futures) - # checking for any exceptions - for future in futures: - if future.exception(): - logger.error(f"Error in thread: {future.exception()}") - - -if __name__ == "__main__": - try: - etsy_api_parser() - except Exception as e: - logger.error(f"Error on fetching orders {e}") diff --git a/src/schemes/access_token.py b/src/schemes/access_token.py deleted file mode 100644 index 0a38438..0000000 --- a/src/schemes/access_token.py +++ /dev/null @@ -1,7 +0,0 @@ -from pydantic import BaseModel - - -class AuthToken(BaseModel): - access_token: str - refresh_token: str - expires_at: float diff --git a/src/schemes/parser_info.py b/src/schemes/parser_info.py deleted file mode 100644 index 1ec813b..0000000 --- a/src/schemes/parser_info.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic import BaseModel - - -class Parser(BaseModel): - id: int - shop_id: int - status: int - command: int - last_parsed: str diff --git a/src/utils/excel/__init__.py b/src/utils/excel/__init__.py deleted file mode 100644 index e69de29..0000000 From 88cf582d2374363458599ab34e40e18c715dd1ab Mon Sep 17 00:00:00 2001 From: Trydimas Date: Fri, 25 Apr 2025 16:40:18 +0300 Subject: [PATCH 13/17] slim refactor files --- src/amazon_api/get_amazon_api.py | 10 +- src/api/order.py | 3 +- src/configs/__init__.py | 2 +- src/configs/env.py | 4 +- src/parser.py | 133 +++++++++----------------- src/parser_all.py | 158 +++++++++---------------------- src/parser_debug.py | 2 +- src/schemes/auth.py | 3 - src/utils/format_order_data.py | 4 +- src/utils/parser_shops_data.py | 7 +- 10 files changed, 108 insertions(+), 218 deletions(-) diff --git a/src/amazon_api/get_amazon_api.py b/src/amazon_api/get_amazon_api.py index 4aee8ad..b3e7619 100644 --- a/src/amazon_api/get_amazon_api.py +++ b/src/amazon_api/get_amazon_api.py @@ -1,17 +1,17 @@ from sp_api.api import Orders from sp_api.util import throttle_retry, load_all_pages from sp_api.base import ApiResponse + import pprint -from configs import settings from constants.amazon_credentials import CREDENTIALS_ARG from constants.status import ParserStatus from utils.safe_ratelimit_amazon import safe_rate_limit -from api.parser import update_parser_status_by_id -from log.logger import logger +from utils.format_order_data import format_order_data from schemes.shop_data import ShopData from schemes.upload_order import OrderData -from utils.format_order_data import format_order_data +from api.parser import update_parser_status_by_id +from log.logger import logger class OrderClient: @@ -46,7 +46,7 @@ def get_orders_with_items(self, page: ApiResponse) -> list[OrderData] | None: try: for order in page.payload.get('Orders'): _order_id = order["AmazonOrderId"] - pprint.pprint(f"formating order ID: {_order_id}") + logger.info(f"formating order ID: {_order_id}") order_data = format_order_data( order=order, items=self._get_all_items(order_id=_order_id) diff --git a/src/api/order.py b/src/api/order.py index b5b7660..70f527f 100644 --- a/src/api/order.py +++ b/src/api/order.py @@ -10,13 +10,14 @@ @retry() def upload_orders_data(orders: UploadingOrderData): + logger.info("Posting data to backend...") response = req.post( f"{settings.API_URL}/parser/orders/upload/", headers=authorization().model_dump(), json=orders.model_dump(), ) - if response.status_code != 200: # TODO поиграться с проверкой + if response.status_code != 200: logger.error( f""" Some error when uploading orders data, status code: {response.status_code} diff --git a/src/configs/__init__.py b/src/configs/__init__.py index f0203a5..f4cc55b 100644 --- a/src/configs/__init__.py +++ b/src/configs/__init__.py @@ -1,5 +1,5 @@ from .env import Settings +settings = Settings() #type: ignore -settings = Settings() diff --git a/src/configs/env.py b/src/configs/env.py index dcfbf5d..ede3db3 100644 --- a/src/configs/env.py +++ b/src/configs/env.py @@ -14,4 +14,6 @@ class Settings(BaseSettings): LOG_FILE: str DATA_FOLDER_PATH: str - + @property + def SHOPS_DATA_FILE_PATH(self): + return f"{self.DATA_FOLDER_PATH}/shops/shops_amazon.json" diff --git a/src/parser.py b/src/parser.py index bc48206..ca6a45c 100644 --- a/src/parser.py +++ b/src/parser.py @@ -1,110 +1,68 @@ import concurrent.futures -import pprint +import json import time -from datetime import datetime, timedelta +from datetime import datetime, timezone -from loguru import logger as log - -from api.order import upload_orders_data from api.parser import update_parser_status_by_id -from configs.env import LOG_FILE +from api.order import upload_orders_data from constants.status import ParserStatus -from etsy_api.get_etsy_api import get_etsy_api -from etsy_api.orders import get_all_orders_by_shop_id -from schemes.upload_order import UploadingOrderData, OrderData -from utils.format_order_data import format_order_data +from schemes.shop_data import ShopData +from schemes.upload_order import UploadingOrderData from utils.parser_shops_data import get_parser_shops_data +from amazon_api.get_amazon_api import OrderClient -log.add( - LOG_FILE, - format="{time} {level} {message}", - level="DEBUG", - rotation="100 MB", - compression="zip", - serialize=True, -) +from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE +from log.logger import logger -# Every 30 minutes PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 -def process_single_shop(shop): +def process_single_shop(shop: ShopData): + order_cl = OrderClient(shop=shop) + created_after = LAST_MONTH_DATE + shop_error = False + offset = 0 + start_time_shop = datetime.now() + weekday = datetime.now().weekday() now_hour = datetime.now().hour - shop_error = False + logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") + logger.info( + f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." + ) - start_time_shop = datetime.now() - log.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") - log.info(f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}...") update_parser_status_by_id( parser_id=shop.parser_id, status=ParserStatus.PARSING, ) - log.success(f"Parser status updated.") + logger.success(f"Parser status updated.") - # Initializing constants - that_month = True - offset = 0 - date = datetime.now() - timedelta(days=30) - weekday = datetime.now().weekday() - ########################## + for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): + """Every 100 orders after """ - while that_month: uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) - log.info( + logger.info( f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." ) - try: - shop_orders, _ = get_all_orders_by_shop_id( - etsy_shop_id=int(shop.etsy_shop_id), - shop_id=shop.shop_id, - limit=100, - offset=offset, - ) - except Exception as e: - log.critical(f"Some error in getting info from ETSY API: {e}") - pprint.pprint(e) - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.ETSY_API_ERROR, - ) + + orders_data = order_cl.get_orders_with_items(page=page_orders) + if orders_data is None: shop_error = True break - # Get order details and split for creating and updating - for shop_order in shop_orders: + uploading_orders.orders_data.extend(orders_data) - order, goods_in_order, day, month, client, city = format_order_data( - order=shop_order, - ) - if day <= date.day and month == date.month: - that_month = False - break - - uploading_orders.orders_data.append( - OrderData( - order=order, - client=client, - city=city, - order_items=goods_in_order, - ) - ) - number_of_attempts = 0 - while number_of_attempts < 10: - res = upload_orders_data(uploading_orders) - if res: - break - number_of_attempts += 1 - if number_of_attempts == 10: - log.critical(f"Some error on sending info to backend") + try: + upload_orders_data(uploading_orders) # upload data to backend + except: + logger.critical(f"Some error on sending info to backend") update_parser_status_by_id( parser_id=shop.parser_id, status=ParserStatus.ETSY_API_ERROR, ) shop_error = True - break offset += 100 @@ -114,9 +72,9 @@ def process_single_shop(shop): break if shop_error: - log.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") + logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") return - log.info( + logger.info( f"Updating parser {shop.parser_id} status to {ParserStatus.OK_AND_WAIT}..." ) @@ -126,32 +84,29 @@ def process_single_shop(shop): last_parsed=datetime.now().timestamp(), ) - log.success(f"Parser status updated.") - log.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") + logger.success(f"Parser status updated.") + logger.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") end_time_shop = datetime.now() - log.info( + logger.info( f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" ) def etsy_api_parser(): shops_data = get_parser_shops_data() - # for shop in shops_data: - # etsy_api = get_etsy_api(shop.shop_id) - # Используем ThreadPoolExecutor для параллельной обработки + + # using ThreadPoolExecutor for parallel processing with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Запускаем обработку каждого магазина в отдельном потоке + # Starting process for each shop in a separate thread futures = [executor.submit(process_single_shop, shop) for shop in shops_data] - # Ждем завершения всех задач + # Waiting for all tasks to complete concurrent.futures.wait(futures) - # Проверяем, были ли исключения + # checking for any exceptions for future in futures: if future.exception(): - log.error(f"Error in thread: {future.exception()}") - log.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") - + logger.error(f"Error in thread: {future.exception()}") + logger.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") -# TODO: сделать чтобы файлы с кредами не писались в файл в многопотоке if __name__ == "__main__": while True: @@ -159,5 +114,5 @@ def etsy_api_parser(): etsy_api_parser() time.sleep(PARSER_WAIT_TIME_IN_SECONDS) except Exception as e: - log.error(f"Error on fetching orders {e}") + logger.error(f"Error on fetching orders {e}") time.sleep(900) diff --git a/src/parser_all.py b/src/parser_all.py index 274c5b5..293e554 100644 --- a/src/parser_all.py +++ b/src/parser_all.py @@ -1,126 +1,69 @@ import concurrent.futures -import pprint -import time from datetime import datetime -from loguru import logger as log - -from api.order import upload_orders_data from api.parser import update_parser_status_by_id -from configs.env import LOG_FILE +from api.order import upload_orders_data from constants.status import ParserStatus -from etsy_api.orders import get_all_orders_by_shop_id -from schemes.upload_order import UploadingOrderData, OrderData -from utils.format_order_data import format_order_data +from schemes.shop_data import ShopData +from schemes.upload_order import UploadingOrderData from utils.parser_shops_data import get_parser_shops_data +from amazon_api.get_amazon_api import OrderClient -log.add( - LOG_FILE, - format="{time} {level} {message}", - level="DEBUG", - rotation="100 MB", - compression="zip", - serialize=True, -) - -# Every 30 minutes -PARSER_WAIT_TIME_IN_SECONDS = 60 * 30 +from constants.amazon_dates import LAST_MONTH_DATE, EARLIEST_DATE, LAST_WEEK_DATE +from log.logger import logger -def process_single_shop(shop): - shops_data = get_parser_shops_data() - now_hour = datetime.now().hour - +def process_single_shop(shop: ShopData): + order_cl = OrderClient(shop=shop) + created_after = LAST_WEEK_DATE shop_error = False - + offset = 0 start_time_shop = datetime.now() - log.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") - log.info( + + logger.info(f"Parsing shop {shop.shop_id} - {shop.shop_name}...") + logger.info( f"Updating parser {shop.parser_id} status to {ParserStatus.PARSING}..." ) + update_parser_status_by_id( parser_id=shop.parser_id, status=ParserStatus.PARSING, ) - log.success(f"Parser status updated.") + logger.success(f"Parser status updated.") - # Initializing constants - offset = 0 - ########################## - try: - shop_orders, orders_count = get_all_orders_by_shop_id( - etsy_shop_id=int(shop.etsy_shop_id), - shop_id=shop.shop_id, - limit=100, - offset=offset, - ) - except Exception as e: - log.critical(f"Some error in getting info from ETSY API: {e}") - pprint.pprint(e) - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.ETSY_API_ERROR, - ) - return - while offset <= orders_count: - uploading_orders = UploadingOrderData(shop_id=shop.shop_id, is_full_data_updating=True, orders_data=[]) + for page_orders in order_cl.load_all_orders(CreatedAfter=created_after): + """Every 100 orders after """ - log.info( + uploading_orders = UploadingOrderData(shop_id=shop.shop_id, orders_data=[]) + + logger.info( f"Fetching orders from {offset} to {offset + 100} from shop {shop.shop_name}..." ) - if offset: - try: - shop_orders, _ = get_all_orders_by_shop_id( - etsy_shop_id=int(shop.etsy_shop_id), - shop_id=shop.shop_id, - limit=100, - offset=offset, - ) - except Exception as e: - log.critical(f"Some error in getting info from ETSY API: {e}") - pprint.pprint(e) - update_parser_status_by_id( - parser_id=shop.parser_id, - status=ParserStatus.ETSY_API_ERROR, - ) - shop_error = True - break - - # Get order details and split for creating and updating - # Get order details and split for creating and updating - for shop_order in shop_orders: - order, goods_in_order, day, month, client, city = format_order_data( - order=shop_order, - ) - uploading_orders.orders_data.append( - OrderData( - order=order, - client=client, - city=city, - order_items=goods_in_order, - ) - ) - number_of_attempts = 0 - while number_of_attempts < 10: - res = upload_orders_data(uploading_orders) - if res: - break - number_of_attempts += 1 - if number_of_attempts == 10: - log.critical(f"Some error on sending info to backend") + + orders_data = order_cl.get_orders_with_items(page=page_orders) + if orders_data is None: + shop_error = True + break + + uploading_orders.orders_data.extend(orders_data) + + try: + upload_orders_data(uploading_orders) # send data to backend + except: + logger.critical(f"Some error on sending info to backend") update_parser_status_by_id( parser_id=shop.parser_id, status=ParserStatus.ETSY_API_ERROR, ) shop_error = True - offset += 100 - if shop_error: - log.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") - return + offset += 100 - log.info( + if shop_error: + logger.error(f"Shop {shop.shop_id} - {shop.shop_name} parsed with error.") + return + logger.info( f"Updating parser {shop.parser_id} status to {ParserStatus.OK_AND_WAIT}..." ) @@ -130,38 +73,31 @@ def process_single_shop(shop): last_parsed=datetime.now().timestamp(), ) - log.success(f"Parser status updated.") - log.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") + logger.success(f"Parser status updated.") + logger.success(f"Shop {shop.shop_id} - {shop.shop_name} parsed.") end_time_shop = datetime.now() - log.info(f"Shop parsing time: {end_time_shop - start_time_shop}") - - log.success( - f"Parsing finished, wait {PARSER_WAIT_TIME_IN_SECONDS} seconds to repeat." + logger.info( + f"Shop {shop.shop_id} - {shop.shop_name} parsing time: {end_time_shop - start_time_shop}" ) def etsy_api_parser(): shops_data = get_parser_shops_data() - # Используем ThreadPoolExecutor для параллельной обработки + # using ThreadPoolExecutor for parallel processing with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Запускаем обработку каждого магазина в отдельном потоке + # Starting process for each shop in a separate thread futures = [executor.submit(process_single_shop, shop) for shop in shops_data] - # Ждем завершения всех задач + # Waiting for all tasks to complete concurrent.futures.wait(futures) - # Проверяем, были ли исключения + # checking for any exceptions for future in futures: if future.exception(): - log.error(f"Error in thread: {future.exception()}") - log.success(f"Parsed all shops waiting {PARSER_WAIT_TIME_IN_SECONDS} to repeat") + logger.error(f"Error in thread: {future.exception()}") if __name__ == "__main__": - try: etsy_api_parser() - time.sleep(PARSER_WAIT_TIME_IN_SECONDS) - time.sleep(900) except Exception as e: - log.error(f"Error on fetching orders {e}") - time.sleep(900) + logger.error(f"Error on fetching orders {e}") diff --git a/src/parser_debug.py b/src/parser_debug.py index b6417b3..46585d1 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -78,4 +78,4 @@ def process_single_shop(shop: ShopData): if __name__ == "__main__": shops_data = get_parser_shops_data() - process_single_shop(shops_data[0]) # TODO пофиксить хуйню с json и магазинами + process_single_shop(shops_data[0]) diff --git a/src/schemes/auth.py b/src/schemes/auth.py index 181dbdd..c002227 100644 --- a/src/schemes/auth.py +++ b/src/schemes/auth.py @@ -4,6 +4,3 @@ class Auth(BaseModel): Authorization: str - -class AuthCode(BaseModel): - code: str diff --git a/src/utils/format_order_data.py b/src/utils/format_order_data.py index 5526757..5299597 100644 --- a/src/utils/format_order_data.py +++ b/src/utils/format_order_data.py @@ -15,7 +15,7 @@ def _format_order(*, order_obj.buyer_paid = None order_obj.order_id = order["AmazonOrderId"] order_obj.status = order["OrderStatus"] # TODO может отличаться от бэка - order_obj.date = iso_to_simple(order["PurchaseDate"]) # formated_date = f"{day}.{month}.{year}" + order_obj.date = iso_to_simple(order["PurchaseDate"]) order_obj.quantity = 0 order_obj.tax = 0 @@ -30,7 +30,7 @@ def _format_good_in_order(*, item_obj.amount = ( (float(item["ItemPrice"]["Amount"]) * item["QuantityOrdered"]) - float(item["PromotionDiscount"]["Amount"]) ) - item_obj.engraving_info = item["Title"] # чё надо? + item_obj.engraving_info = item["Title"] # TODO чё надо? def _format_client(*, diff --git a/src/utils/parser_shops_data.py b/src/utils/parser_shops_data.py index 42b798d..5add1e5 100644 --- a/src/utils/parser_shops_data.py +++ b/src/utils/parser_shops_data.py @@ -1,11 +1,10 @@ -import json - -from constants.files_paths import SHOPS_DATA_FILE_PATH +from configs import settings from schemes.shop_data import ShopData +import json def get_parser_shops_data() -> list[ShopData]: - with open(SHOPS_DATA_FILE_PATH) as f: + with open(settings.SHOPS_DATA_FILE_PATH) as f: shops_data = [ShopData(**shop_data) for shop_data in json.load(f)] return shops_data From e8fd2b573b13a50489cf9bb10b0bb9ea5753ab77 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Sat, 3 May 2025 18:06:25 +0300 Subject: [PATCH 14/17] add: order statuses --- src/api/order.py | 2 +- src/configs/env.py | 5 +++++ src/constants/status.py | 43 ++++++++++++++++++++++++++++++++++++ src/utils/format_datetime.py | 7 +++--- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/api/order.py b/src/api/order.py index 70f527f..63d1b2b 100644 --- a/src/api/order.py +++ b/src/api/order.py @@ -12,7 +12,7 @@ def upload_orders_data(orders: UploadingOrderData): logger.info("Posting data to backend...") response = req.post( - f"{settings.API_URL}/parser/orders/upload/", + url=settings.PARSER_ORDER_UPLOAD, headers=authorization().model_dump(), json=orders.model_dump(), ) diff --git a/src/configs/env.py b/src/configs/env.py index ede3db3..bc4c588 100644 --- a/src/configs/env.py +++ b/src/configs/env.py @@ -17,3 +17,8 @@ class Settings(BaseSettings): @property def SHOPS_DATA_FILE_PATH(self): return f"{self.DATA_FOLDER_PATH}/shops/shops_amazon.json" + + + @property + def PARSER_ORDER_UPLOAD(self): + return f"{self.API_URL}/parser/orders/upload/" diff --git a/src/constants/status.py b/src/constants/status.py index a1ee38f..cc499cb 100644 --- a/src/constants/status.py +++ b/src/constants/status.py @@ -3,3 +3,46 @@ class ParserStatus: PARSING = 1 COOKIE_EXPIRED = 2 ETSY_API_ERROR = 3 + + + +class OrderStatus: + Paid = "Paid" + Canceled = "Canceled" + Completed = "Completed" + PartiallyRefunded = "Partially Refunded" + FullyRefunded = "Fully Refunded" + + @classmethod + def get_backend_status(cls, amazon_status): + + #TODO amazon не предоставлыет доступ к инфе о refund, с одним но + + """ + PendingAvailability — Available only for pre-orders. The order has been placed, but the payment has not been authorized, and the release date of the product is in the future. + + Pending — The order has been placed, but the payment has not yet been authorized. The order is not ready for shipment. + + Unshipped — Payment is authorized, the order is ready for shipment, but no product has been shipped yet. + + PartiallyShipped — One or more items have been shipped, but not all. + + Shipped — All items in the order have been shipped. + + InvoiceUnconfirmed — All items have been shipped, but the seller has not yet confirmed to Amazon that the invoice has been sent to the buyer. + + Cancelled — The order has been cancelled. + + Unfulfillable — The order cannot be completed. This status only applies to orders completed by Amazon that have not been placed on the Amazon retail website. + """ + mapping = { + "Pending": cls.Paid, + "Unshipped": cls.Paid, + "PendingAvailability": cls.Paid, + "PartiallyShipped": cls.Completed, + "Shipped": cls.Completed, + "InvoiceUnconfirmed": cls.Completed, + "Canceled": cls.Canceled, + "Unfulfillable": cls.Canceled + } + return mapping.get(amazon_status) \ No newline at end of file diff --git a/src/utils/format_datetime.py b/src/utils/format_datetime.py index 1199c59..f146a69 100644 --- a/src/utils/format_datetime.py +++ b/src/utils/format_datetime.py @@ -12,6 +12,7 @@ def is_iso_utc_z_format(date_str): def iso_to_simple(iso_str: str): """iso8601 to dd.mm.YYYY""" - dt = datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return f"{dt.day:02d}.{dt.month:02d}.{dt.year}" - + if is_iso_utc_z_format(iso_str): + dt = datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return f"{dt.day:02d}.{dt.month:02d}.{dt.year}" + return None From 9a022143eecb614475d5570a82e1defb7b662bcf Mon Sep 17 00:00:00 2001 From: Trydimas Date: Sat, 3 May 2025 18:07:08 +0300 Subject: [PATCH 15/17] fix: format_order_data types --- src/utils/format_order_data.py | 64 +++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/utils/format_order_data.py b/src/utils/format_order_data.py index 5299597..59df3da 100644 --- a/src/utils/format_order_data.py +++ b/src/utils/format_order_data.py @@ -1,54 +1,61 @@ -import json -from datetime import datetime - from schemes.order import Order from schemes.client import Client from schemes.city import City from schemes.order_item import GoodInOrder from schemes.upload_order import OrderData from utils.format_datetime import iso_to_simple +from constants.status import OrderStatus def _format_order(*, order: dict, order_obj: Order): """earliest fill order_obj""" - order_obj.buyer_paid = None - order_obj.order_id = order["AmazonOrderId"] - order_obj.status = order["OrderStatus"] # TODO может отличаться от бэка - order_obj.date = iso_to_simple(order["PurchaseDate"]) + order_obj.buyer_paid = float(order.get("OrderTotal", {}).get("Amount", 0)) + order_obj.order_id = order.get("AmazonOrderId", "") + order_obj.status = OrderStatus.get_backend_status(order.get("OrderStatus")) + order_obj.date = iso_to_simple(order.get("PurchaseDate")) order_obj.quantity = 0 order_obj.tax = 0 + #TODO на бэке сделать ручку под amazon + + def _format_good_in_order(*, item: dict, item_obj: GoodInOrder): """fill good_in_order obj""" - item_obj.uniquename = item["SellerSKU"] - item_obj.quantity = item["QuantityOrdered"] - if item.get("ItemPrice") and item.get("PromotionDiscount"): - item_obj.amount = ( - (float(item["ItemPrice"]["Amount"]) * item["QuantityOrdered"]) - float(item["PromotionDiscount"]["Amount"]) + + item_price = item.get("ItemPrice") + item_quantity= item.get("QuantityOrdered") + item_discount = item.get("PromotionDiscount") + + item_obj.uniquename = item.get("SellerSKU") + item_obj.quantity = item_quantity + + _amount = 0.0 + if item_price and item_discount and item_quantity: + _amount = ( + (float(item_price["Amount"]) * item_quantity) - float(item_discount["Amount"]) ) - item_obj.engraving_info = item["Title"] # TODO чё надо? + + item_obj.amount = _amount #TODO Отменённые заказы имеют quantity 0 при этом стоимость item и TotalOrder могут меть значения. проблема в подсчёте amount + item_obj.engraving_info = item["Title"] # TODO На перспективу def _format_client(*, order: dict, client_obj: Client): - buyer_info = order.get("BuyerInfo") - if buyer_info: - client_obj.email = buyer_info["BuyerEmail"] + client_obj.email = order.get("BuyerInfo", {}).get("BuyerEmail") def _format_city(*, order: dict, city_obj: City): - shipping_address = order.get("ShippingAddress") - if shipping_address: - city_obj.name = shipping_address["City"] - city_obj.state = shipping_address["StateOrRegion"] - city_obj.country = shipping_address["CountryCode"] + shipping_address = order.get("ShippingAddress", {}) + city_obj.name = shipping_address.get("City") + city_obj.state = shipping_address.get("StateOrRegion") + city_obj.country = shipping_address.get("CountryCode") def format_order_data(*, @@ -68,12 +75,19 @@ def format_order_data(*, _format_good_in_order(item=item, item_obj=good_in_order) # calculate tax and quantity for order - order_obj.quantity += good_in_order.quantity + item_quantity = good_in_order.quantity + item_tax = item.get("ItemTax") + item_discount_tax = item.get("PromotionDiscountTax") + + order_obj.quantity += item_quantity - if item.get("ItemTax") and item.get("PromotionDiscountTax"): - order_obj.tax += ( - (good_in_order.quantity * float(item["ItemTax"]["Amount"])) - float(item["PromotionDiscountTax"]["Amount"]) + # TODO Отменённые заказы имеют quantity 0 при этом стоимость item и TotalOrder могут меть значения. проблема в подсчёте amount + _amount_tax = 0.0 + if item_tax and item_discount_tax and item_quantity: + _amount_tax += ( + (item_quantity * float(item_tax["Amount"])) - float(item_discount_tax["Amount"]) ) + order_obj.tax = _amount_tax order_items.append(good_in_order) From cd3a45ea5457b1dc984df32e43de56d7ea7f13d4 Mon Sep 17 00:00:00 2001 From: Trydimas Date: Sun, 4 May 2025 17:41:26 +0300 Subject: [PATCH 16/17] slim refactor files --- src/api/parser.py | 4 +++- src/constants/status.py | 2 +- src/parser_debug.py | 2 +- src/utils/format_order_data.py | 6 ++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/api/parser.py b/src/api/parser.py index 88b8a60..d89ad18 100644 --- a/src/api/parser.py +++ b/src/api/parser.py @@ -6,7 +6,9 @@ def update_parser_status_by_id( - parser_id: int, status: int, last_parsed: float | None = None + parser_id: int, + status: int, + last_parsed: float | None = None ): data = { "id": parser_id, diff --git a/src/constants/status.py b/src/constants/status.py index cc499cb..b35e58f 100644 --- a/src/constants/status.py +++ b/src/constants/status.py @@ -41,7 +41,7 @@ def get_backend_status(cls, amazon_status): "PendingAvailability": cls.Paid, "PartiallyShipped": cls.Completed, "Shipped": cls.Completed, - "InvoiceUnconfirmed": cls.Completed, + "InvoiceUnconfirmed": cls.Paid, "Canceled": cls.Canceled, "Unfulfillable": cls.Canceled } diff --git a/src/parser_debug.py b/src/parser_debug.py index 46585d1..18ff1fd 100644 --- a/src/parser_debug.py +++ b/src/parser_debug.py @@ -49,7 +49,7 @@ def process_single_shop(shop: ShopData): uploading_orders.orders_data.extend(orders_data) - with open("test_data.json", "w") as f: + with open("test_data_2.json", "w") as f: json.dump(uploading_orders.model_dump(), f) offset += 100 diff --git a/src/utils/format_order_data.py b/src/utils/format_order_data.py index 59df3da..02d9ee8 100644 --- a/src/utils/format_order_data.py +++ b/src/utils/format_order_data.py @@ -39,8 +39,8 @@ def _format_good_in_order(*, (float(item_price["Amount"]) * item_quantity) - float(item_discount["Amount"]) ) - item_obj.amount = _amount #TODO Отменённые заказы имеют quantity 0 при этом стоимость item и TotalOrder могут меть значения. проблема в подсчёте amount - item_obj.engraving_info = item["Title"] # TODO На перспективу + item_obj.amount = _amount + item_obj.engraving_info = item["Title"] # TODO СДЕЛАТЬ!!!! def _format_client(*, @@ -48,7 +48,6 @@ def _format_client(*, client_obj: Client): client_obj.email = order.get("BuyerInfo", {}).get("BuyerEmail") - def _format_city(*, order: dict, city_obj: City): @@ -81,7 +80,6 @@ def format_order_data(*, order_obj.quantity += item_quantity - # TODO Отменённые заказы имеют quantity 0 при этом стоимость item и TotalOrder могут меть значения. проблема в подсчёте amount _amount_tax = 0.0 if item_tax and item_discount_tax and item_quantity: _amount_tax += ( From a1befbf6fd412d8358330b1637ff33cccf0fdf2c Mon Sep 17 00:00:00 2001 From: Trydimas Date: Mon, 5 May 2025 01:59:02 +0300 Subject: [PATCH 17/17] slim refactor files --- src/api/order.py | 2 +- src/api/parser.py | 4 +++- src/configs/env.py | 8 ++++++-- src/constants/status.py | 3 --- src/parser_all.py | 2 +- src/utils/format_order_data.py | 3 --- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/api/order.py b/src/api/order.py index 63d1b2b..313a497 100644 --- a/src/api/order.py +++ b/src/api/order.py @@ -12,7 +12,7 @@ def upload_orders_data(orders: UploadingOrderData): logger.info("Posting data to backend...") response = req.post( - url=settings.PARSER_ORDER_UPLOAD, + url=settings.PARSER_ORDER_UPLOAD_URL, headers=authorization().model_dump(), json=orders.model_dump(), ) diff --git a/src/api/parser.py b/src/api/parser.py index d89ad18..e3e08de 100644 --- a/src/api/parser.py +++ b/src/api/parser.py @@ -20,7 +20,9 @@ def update_parser_status_by_id( try: response = req.put( - f"{settings.API_URL}/parser/", headers=authorization().model_dump(), json=data + url=settings.PARSER_STATUS_URL, + headers=authorization().model_dump(), + json=data ) if response.status_code != 200: diff --git a/src/configs/env.py b/src/configs/env.py index bc4c588..87b2c70 100644 --- a/src/configs/env.py +++ b/src/configs/env.py @@ -20,5 +20,9 @@ def SHOPS_DATA_FILE_PATH(self): @property - def PARSER_ORDER_UPLOAD(self): - return f"{self.API_URL}/parser/orders/upload/" + def PARSER_ORDER_UPLOAD_URL(self): + return f"{self.API_URL}/parser/orders/upload/amazon" + + @property + def PARSER_STATUS_URL(self): + return f"{self.API_URL}/parser/" \ No newline at end of file diff --git a/src/constants/status.py b/src/constants/status.py index b35e58f..2df8256 100644 --- a/src/constants/status.py +++ b/src/constants/status.py @@ -5,7 +5,6 @@ class ParserStatus: ETSY_API_ERROR = 3 - class OrderStatus: Paid = "Paid" Canceled = "Canceled" @@ -16,8 +15,6 @@ class OrderStatus: @classmethod def get_backend_status(cls, amazon_status): - #TODO amazon не предоставлыет доступ к инфе о refund, с одним но - """ PendingAvailability — Available only for pre-orders. The order has been placed, but the payment has not been authorized, and the release date of the product is in the future. diff --git a/src/parser_all.py b/src/parser_all.py index 293e554..02106c2 100644 --- a/src/parser_all.py +++ b/src/parser_all.py @@ -15,7 +15,7 @@ def process_single_shop(shop: ShopData): order_cl = OrderClient(shop=shop) - created_after = LAST_WEEK_DATE + created_after = EARLIEST_DATE shop_error = False offset = 0 start_time_shop = datetime.now() diff --git a/src/utils/format_order_data.py b/src/utils/format_order_data.py index 02d9ee8..08addb6 100644 --- a/src/utils/format_order_data.py +++ b/src/utils/format_order_data.py @@ -18,9 +18,6 @@ def _format_order(*, order_obj.tax = 0 - #TODO на бэке сделать ручку под amazon - - def _format_good_in_order(*, item: dict, item_obj: GoodInOrder):