import os import time import math import itertools import traceback from concurrent.futures import ThreadPoolExecutor, as_completed import requests from django.core.management.base import BaseCommand from django.conf import settings from django.db.models import Q from recharge.models import Operator, Circle DEFAULT_BASE_URL = getattr(settings, "VAP_BASE_URL", "https://vap.proactiveevents.in") PLANS_ENDPOINT = "/api/recharge/plans/" def build_url(base_url, operator_code, circle_code, provider=None, service="prepaid", refresh=False): params = { "operator": operator_code, "circle": circle_code, "service": service, # harmless for your PlansAPI; kept for parity with your example } if provider: params["provider"] = provider if refresh: params["refresh"] = "1" # Build query string qs = "&".join(f"{k}={requests.utils.quote(str(v))}" for k, v in params.items()) if not base_url.endswith("/"): base_url += "/" return f"{base_url.rstrip('/')}{PLANS_ENDPOINT}?{qs}" def hit_once(session, url, timeout): try: r = session.get(url, timeout=timeout) return r.status_code, r.json() except Exception as e: return None, {"error": str(e), "trace": traceback.format_exc()} class Command(BaseCommand): help = "Fetch all recharge plans for operator/circle combinations via your public Plans API and warm the PlanCache." def add_arguments(self, parser): parser.add_argument("--base-url", type=str, default=DEFAULT_BASE_URL, help=f"Base site URL hosting the API (default: {DEFAULT_BASE_URL})") parser.add_argument("--provider", type=str, default="goterpay", help="Provider query value (default: goterpay); your API maps this to source") parser.add_argument("--source", type=str, default="Goter", help="Filter Operators/Circles by source (default: Goter)") parser.add_argument("--service-type", type=str, default="Mobile", help="Filter Operators by service_type (default: Mobile)") parser.add_argument("--only-operators", type=str, default="", help="Comma-separated operator codes to include (e.g. AT,VI,JO). If set, ignore others.") parser.add_argument("--only-circles", type=str, default="", help="Comma-separated circle codes to include (e.g. 1,2,KA). If set, ignore others.") parser.add_argument("--exclude-operators", type=str, default="", help="Comma-separated operator codes to skip.") parser.add_argument("--exclude-circles", type=str, default="", help="Comma-separated circle codes to skip.") parser.add_argument("--refresh", action="store_true", help="Pass refresh=1 so API refetches from provider and updates cache.") parser.add_argument("--timeout", type=int, default=25, help="HTTP timeout per request (seconds). Default: 25") parser.add_argument("--throttle-ms", type=int, default=120, help="Sleep this many milliseconds between starts (per worker). Default: 120ms") parser.add_argument("--workers", type=int, default=8, help="Number of parallel workers (threads). Default: 8") parser.add_argument("--dry-run", action="store_true", help="Print what would be called, but do not perform HTTP requests.") def handle(self, *args, **opts): base_url = opts["base_url"].rstrip("/") provider_q = opts["provider"] source = opts["source"] service_type = opts["service_type"] only_ops = {x.strip() for x in opts["only_operators"].split(",") if x.strip()} only_circles = {x.strip() for x in opts["only_circles"].split(",") if x.strip()} excl_ops = {x.strip() for x in opts["exclude_operators"].split(",") if x.strip()} excl_circles = {x.strip() for x in opts["exclude_circles"].split(",") if x.strip()} refresh = bool(opts["refresh"]) timeout = int(opts["timeout"]) throttle_ms = int(opts["throttle_ms"]) workers = max(1, int(opts["workers"])) dry_run = bool(opts["dry_run"]) self.stdout.write(self.style.NOTICE( f"Fetching plans via {base_url}{PLANS_ENDPOINT} " f"[provider={provider_q}, service=prepaid, refresh={int(refresh)}]" )) # Operators filter op_qs = Operator.objects.filter(source__iexact=source) if service_type: op_qs = op_qs.filter(service_type__iexact=service_type) if only_ops: op_qs = op_qs.filter(code__in=only_ops) if excl_ops: op_qs = op_qs.exclude(code__in=excl_ops) # Circles filter circle_qs = Circle.objects.filter(source__iexact=source) if only_circles: circle_qs = circle_qs.filter(code__in=only_circles) if excl_circles: circle_qs = circle_qs.exclude(code__in=excl_circles) operators = list(op_qs.values_list("code", flat=True).order_by("code").distinct()) circles = list(circle_qs.values_list("code", flat=True).order_by("code").distinct()) if not operators: self.stdout.write(self.style.WARNING("No operators matched filters. Exiting.")) return if not circles: self.stdout.write(self.style.WARNING("No circles matched filters. Exiting.")) return combos = list(itertools.product(operators, circles)) total = len(combos) self.stdout.write(self.style.NOTICE(f"Planned requests: {total} ({len(operators)} operators × {len(circles)} circles)")) if dry_run: for op, ci in combos: url = build_url(base_url, op, ci, provider=provider_q, service="prepaid", refresh=refresh) self.stdout.write(f"[DRY] {url}") self.stdout.write(self.style.SUCCESS("Dry-run complete.")) return ok = 0 failed = 0 served_from_cache = 0 fresh = 0 errors = [] session = requests.Session() start_ts = time.time() def worker(idx, op, ci): # gentle staggering to avoid spikes if throttle_ms > 0: time.sleep((throttle_ms / 1000.0) * (idx % max(1, workers))) url = build_url(base_url, op, ci, provider=provider_q, service="prepaid", refresh=refresh) status_code, payload = hit_once(session, url, timeout=timeout) meta = { "operator": op, "circle": ci, "url": url, "status_code": status_code, } if status_code != 200: meta["result"] = "HTTP_ERROR" meta["detail"] = str(payload)[:500] return meta # Expect your API contract: { status, plans, raw, cache: { served_from_cache } } try: status_val = str(payload.get("status", "")).upper() cache_info = payload.get("cache") or {} sfc = bool(cache_info.get("served_from_cache")) plans_len = len(payload.get("plans") or []) except Exception: status_val = "PARSE_ERROR" sfc = False plans_len = 0 meta["result"] = status_val meta["served_from_cache"] = sfc meta["plans_len"] = plans_len return meta with ThreadPoolExecutor(max_workers=workers) as ex: futures = {} for idx, (op, ci) in enumerate(combos, start=1): futures[ex.submit(worker, idx, op, ci)] = (op, ci) processed = 0 for fut in as_completed(futures): processed += 1 res = fut.result() op = res.get("operator") ci = res.get("circle") if res.get("result") in ("SUCCESS", "OK"): ok += 1 if res.get("served_from_cache"): served_from_cache += 1 else: fresh += 1 self.stdout.write(f"✓ [{processed}/{total}] {op}-{ci} : {res.get('plans_len', 0)} plans " f"({'cache' if res.get('served_from_cache') else 'fresh'})") else: failed += 1 errors.append(res) code = res.get("status_code") detail = res.get("detail", "")[:200] self.stdout.write(self.style.WARNING( f"× [{processed}/{total}] {op}-{ci} : {res.get('result')} (HTTP {code}) {detail}" )) dur = time.time() - start_ts self.stdout.write("") self.stdout.write(self.style.SUCCESS( f"Done in {dur:.1f}s | OK: {ok} (fresh: {fresh}, cache: {served_from_cache}) | Failed: {failed} | Total: {total}" )) if errors: self.stdout.write(self.style.WARNING("Some requests failed; first few:")) for e in errors[:10]: self.stdout.write(f"- {e.get('operator')}-{e.get('circle')} " f"HTTP {e.get('status_code')} result={e.get('result')} url={e.get('url')}")