import csv
import datetime
import io
import json
import os
import re
import sys
import urllib.request
USER_AGENT = "Mozilla/5.0 (compatible; leetctl-gen-sets/1.0)"
LEETCODE_PROBLEM_INDEX = "https://leetcode.com/api/problems/all/"
LEETCODE_GRAPHQL = "https://leetcode.com/graphql"
NEETCODE_SITE_DATA = (
"https://raw.githubusercontent.com/neetcode-gh/leetcode/main/.problemSiteData.json"
)
NEETCODE_PRACTICE_PAGE = "https://neetcode.io/practice/practice/neetcode250"
NEETCODE_BUNDLE_BASE = "https://neetcode.io/"
COMPANY_CSV_BASE = (
"https://raw.githubusercontent.com/hxu296/"
"leetcode-company-wise-problems-2022/main/companies"
)
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUTPUT_DIR = os.path.join(REPO_ROOT, "data", "sets")
NEETCODE_SOURCE = {
"source_url": "https://github.com/neetcode-gh/leetcode",
"source_license": "MIT",
"source_as_of": None, }
NEETCODE_BUNDLE_SOURCE = {
"source_url": "https://neetcode.io/practice/practice/neetcode250",
"source_license": "NeetCode list membership; problem identifiers only",
"source_as_of": None,
}
LEETCODE_SOURCE = {
"source_url": "https://leetcode.com/studyplan/",
"source_license": "Proprietary (LeetCode); problem identifiers only",
"source_as_of": None,
}
COMPANY_SOURCE = {
"source_url": "https://github.com/hxu296/leetcode-company-wise-problems-2022",
"source_license": "MIT",
"source_as_of": "2022",
}
NEETCODE_SETS = [
("blind75", "Blind 75", "The original Blind 75 list, as tracked by NeetCode.", "blind75"),
(
"neetcode150",
"NeetCode 150",
"NeetCode's 150-problem core list, a superset of Blind 75.",
"neetcode150",
),
(
"neetcode-all",
"NeetCode All",
"Every problem NeetCode publishes a solution for, grouped into 19 patterns.",
None,
),
]
LEETCODE_STUDY_PLANS = [
(
"top-interview-150",
"Top Interview 150",
"LeetCode's official Top Interview 150 study plan.",
),
(
"top-100-liked",
"Top 100 Liked",
"LeetCode's official Top 100 Liked study plan.",
),
(
"leetcode-75",
"LeetCode 75",
"LeetCode's official 75-problem starter study plan.",
),
]
RENAMED_SLUGS = {
"coin-change-2": "coin-change-ii", "increasing-subsequences": "non-decreasing-subsequences", "add-and-search-word-data-structure-design":
"design-add-and-search-words-data-structure", "implement-strstr": "find-the-index-of-the-first-occurrence-in-a-string", "make-two-arrays-equal-by-reversing-sub-arrays":
"make-two-arrays-equal-by-reversing-subarrays", "friend-circles": "number-of-provinces", "bulb-switcher-iii": "number-of-times-binary-string-is-prefix-aligned", }
COMPANIES = [
("google", "Google", "Google"),
("facebook", "Meta / Facebook", "Facebook"),
("amazon", "Amazon", "Amazon"),
("microsoft", "Microsoft", "Microsoft"),
]
EXPECTED_COUNTS = {
"blind75": 75,
"neetcode150": 150,
"neetcode250": 250,
"neetcode-all": 450,
"top-interview-150": 150,
"top-100-liked": 100,
"leetcode-75": 75,
"google": 488,
"facebook": 371,
"amazon": 592,
"microsoft": 363,
}
KNOWN_OMISSIONS = {}
def fetch(url, data=None):
request = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT})
if data is not None:
request.add_header("Content-Type", "application/json")
with urllib.request.urlopen(request, timeout=60) as response:
return response.read()
def fetch_json(url, payload=None):
body = json.dumps(payload).encode() if payload is not None else None
return json.loads(fetch(url, body))
def slug_to_fid_index():
index = fetch_json(LEETCODE_PROBLEM_INDEX)
return {
pair["stat"]["question__title_slug"]: int(pair["stat"]["frontend_question_id"])
for pair in index["stat_status_pairs"]
}
def slug_from_problem_url(url):
return url.rstrip("/").rsplit("/", 1)[-1]
def neetcode_bundle_problems():
page = fetch(NEETCODE_PRACTICE_PAGE).decode("utf-8", "replace")
match = re.search(r'src="(main\.[0-9a-f]+\.js)"', page)
if match is None:
raise GenerationError(
"could not find the main bundle on the NeetCode practice page; the site's markup "
"changed and neetcode_bundle_problems() needs updating"
)
bundle = fetch(NEETCODE_BUNDLE_BASE + match.group(1)).decode("utf-8", "replace")
problems = []
for start in (m.start() for m in re.finditer(r'\{problem:"', bundle)):
depth = 0
for index in range(start, min(start + 4000, len(bundle))):
if bundle[index] == "{":
depth += 1
elif bundle[index] == "}":
depth -= 1
if depth == 0:
problems.append(bundle[start:index + 1])
break
def slug_of(literal):
found = re.search(r'\blink:"((?:[^"\\]|\\.)*)"', literal)
return found.group(1).strip("/") if found else None
parsed = []
for literal in problems:
slug = slug_of(literal)
if not slug:
continue
parsed.append({
"slug": slug,
"blind75": re.search(r"\bblind75:!0", literal) is not None,
"neetcode150": re.search(r"\bneetcode150:!0", literal) is not None,
"neetcode250": re.search(r"\bneetcode250:!0", literal) is not None,
})
if len(parsed) < 500:
raise GenerationError(
f"only {len(parsed)} problems parsed out of the NeetCode bundle, expected ~1000; "
f"the bundle's shape changed"
)
return parsed
def check_neetcode_sources_agree(bundle, site_data):
for flag in ("blind75", "neetcode150"):
from_bundle = {p["slug"] for p in bundle if p[flag]}
from_file = {p["link"].strip("/") for p in site_data if p.get(flag)}
if from_bundle != from_file:
difference = ", ".join(sorted(from_bundle ^ from_file)[:10])
raise GenerationError(
f"NeetCode's website bundle and MIT data file disagree about `{flag}` "
f"({len(from_bundle)} vs {len(from_file)} problems; differing: {difference}). "
f"They have drifted apart — pick one source for all NeetCode sets."
)
blind = {p["slug"] for p in bundle if p["blind75"]}
hundred_fifty = {p["slug"] for p in bundle if p["neetcode150"]}
two_fifty = {p["slug"] for p in bundle if p["neetcode250"]}
if not (blind <= hundred_fifty <= two_fifty):
raise GenerationError(
"NeetCode's lists no longer nest (blind75 in neetcode150 in neetcode250); "
"the site changed how its lists relate"
)
def neetcode_slugs(site_data, membership_flag):
return [
entry["link"].strip("/")
for entry in site_data
if membership_flag is None or entry.get(membership_flag)
]
def study_plan_slugs(plan_slug):
query = (
"query($slug: String!) { studyPlanV2Detail(planSlug: $slug) "
"{ planSubGroups { questions { titleSlug } } } }"
)
payload = {"query": query, "variables": {"slug": plan_slug}}
detail = fetch_json(LEETCODE_GRAPHQL, payload)["data"]["studyPlanV2Detail"]
if detail is None:
raise SystemExit(f"study plan {plan_slug!r} returned no detail")
return [
question["titleSlug"]
for group in detail["planSubGroups"]
for question in group["questions"]
]
def company_slugs(csv_name):
text = fetch(f"{COMPANY_CSV_BASE}/{csv_name}.csv").decode()
rows = csv.DictReader(io.StringIO(text))
return [slug_from_problem_url(row["problem_link"]) for row in rows]
def toml_string(value):
return json.dumps(value, ensure_ascii=False)
class GenerationError(Exception):
def resolve(slug, problem_slugs, fid_index):
resolved = []
unresolved = []
seen_fids = set()
for problem_slug in problem_slugs:
problem_slug = RENAMED_SLUGS.get(problem_slug, problem_slug)
if problem_slug in KNOWN_OMISSIONS.get(slug, ()):
continue
fid = fid_index.get(problem_slug)
if fid is None:
unresolved.append(problem_slug)
continue
if fid in seen_fids:
continue
seen_fids.add(fid)
resolved.append((fid, problem_slug))
if unresolved:
raise GenerationError(
f"{slug}: {len(unresolved)} slug(s) do not resolve to a LeetCode frontend id: "
f"{', '.join(unresolved)}.\n"
f" Either LeetCode renamed them — add the new slug to RENAMED_SLUGS, verified "
f"against the live index by frontend id — or they were removed, in which case add "
f"them to KNOWN_OMISSIONS[{slug!r}] with a reason."
)
expected = EXPECTED_COUNTS[slug]
if len(resolved) != expected:
raise GenerationError(
f"{slug}: resolved {len(resolved)} problems, expected {expected}. "
f"The source changed. Review the diff, then update EXPECTED_COUNTS."
)
return resolved
def render_set(slug, name, description, source, problem_slugs, fid_index, sort_by_fid):
resolved = resolve(slug, problem_slugs, fid_index)
if sort_by_fid:
resolved.sort()
generated_at = datetime.date.today().isoformat()
lines = [
"# @generated by scripts/gen_sets.py — do not edit by hand.",
f"slug = {toml_string(slug)}",
f"name = {toml_string(name)}",
f"description = {toml_string(description)}",
f"source_url = {toml_string(source['source_url'])}",
f"source_license = {toml_string(source['source_license'])}",
f"source_as_of = {toml_string(source['source_as_of'] or generated_at)}",
f"generated_at = {toml_string(generated_at)}",
"",
"problems = [",
]
lines += [f" {{ fid = {fid}, slug = {toml_string(s)} }}," for fid, s in resolved]
lines += ["]", ""]
return slug, len(resolved), "\n".join(lines)
def write_all(rendered):
for slug, count, contents in rendered:
path = os.path.join(OUTPUT_DIR, f"{slug}.toml")
temporary = f"{path}.tmp"
with open(temporary, "w", encoding="utf-8") as handle:
handle.write(contents)
os.replace(temporary, path)
print(f"{slug:<20} {count:>4} problems")
def main():
os.makedirs(OUTPUT_DIR, exist_ok=True)
print("fetching LeetCode problem index...")
fid_index = slug_to_fid_index()
print(f" {len(fid_index)} problems indexed")
print("fetching NeetCode site data...")
site_data = fetch_json(NEETCODE_SITE_DATA)
print(f" {len(site_data)} problems")
print("fetching NeetCode website bundle...")
bundle = neetcode_bundle_problems()
print(f" {len(bundle)} problems")
check_neetcode_sources_agree(bundle, site_data)
rendered = []
try:
for slug, name, description, flag in NEETCODE_SETS:
rendered.append(render_set(
slug, name, description, NEETCODE_SOURCE,
neetcode_slugs(site_data, flag), fid_index, sort_by_fid=False,
))
rendered.append(render_set(
"neetcode250", "NeetCode 250",
"NeetCode's 250-problem list, a superset of NeetCode 150.",
NEETCODE_BUNDLE_SOURCE,
[p["slug"] for p in bundle if p["neetcode250"]], fid_index, sort_by_fid=False,
))
for slug, name, description in LEETCODE_STUDY_PLANS:
rendered.append(render_set(
slug, name, description, LEETCODE_SOURCE,
study_plan_slugs(slug), fid_index, sort_by_fid=False,
))
for slug, name, csv_name in COMPANIES:
description = (
f"Problems carrying the {name} tag in a community-collected 2022 snapshot."
)
rendered.append(render_set(
slug, name, description, COMPANY_SOURCE,
company_slugs(csv_name), fid_index, sort_by_fid=True,
))
except GenerationError as error:
print(f"\nerror: {error}\nNothing was written.", file=sys.stderr)
raise SystemExit(1)
write_all(rendered)
if __name__ == "__main__":
main()