#!/usr/bin/env python3
"""Import native eval files, then optionally start Braintrust or open LangSmith."""

# ruff: noqa: E402
# Bootstrap using built-ins only: even importing os/getpass here could execute a
# seller's same-named file beside this helper. Restart before any file-based
# imports so Python excludes the package directory and PYTHONPATH. Environment
# credentials, arguments, and the working directory survive isolated mode.
import sys

if __name__ == "__main__" and not sys.flags.isolated:
    _bootstrap_args = [sys.executable, "-I", __file__, *sys.argv[1:]]
    if sys.platform == "win32":
        # Windows execv loses the child's exit status. spawnv waits, but needs
        # Windows CRT quoting (including quotes and trailing backslashes).
        # Do not import subprocess for list2cmdline before isolation.
        def _quote_windows_argument(value):
            quoted = '"'
            backslashes = 0
            for char in value:
                if char == "\\":
                    backslashes += 1
                    continue
                quoted += "\\" * (backslashes * 2 + 1 if char == '"' else backslashes)
                quoted += char
                backslashes = 0
            return quoted + "\\" * (backslashes * 2) + '"'

        _bootstrap_nt = __import__("nt")
        raise SystemExit(
            _bootstrap_nt.spawnv(
                _bootstrap_nt.P_WAIT,
                sys.executable,
                [_quote_windows_argument(arg) for arg in _bootstrap_args],
            )
        )
    __import__("posix").execv(sys.executable, _bootstrap_args)

# isort: split
import argparse
import copy
import getpass
import json
import math
import os
import re
import shutil
import subprocess
import warnings
import webbrowser
from datetime import datetime, timezone
from pathlib import Path
from time import monotonic
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode, urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
from uuid import UUID, uuid4

BRAINTRUST_FILE_BYTES = 5 * 1024 * 1024
BRAINTRUST_API = "https://api.braintrust.dev"
NUMERIC_PARAMS = {
    "temperature",
    "top_p",
    "top_k",
    "max_tokens",
    "max_completion_tokens",
    "frequency_penalty",
    "presence_penalty",
    "n",
    "reasoning_budget",
    "max_tokens_to_sample",
    "maxOutputTokens",
    "topP",
    "topK",
}
BOOL_PARAMS = {"use_cache", "reasoning_enabled"}


def native_json(text):
    """Reject duplicate keys and non-finite values, including exponent overflow."""

    def pairs(items):
        result = {}
        for key, value in items:
            if key in result:
                raise ValueError("Duplicate JSON object key.")
            result[key] = value
        return result

    value = json.loads(text, object_pairs_hook=pairs)

    def finite(item):
        if isinstance(item, float) and not math.isfinite(item):
            raise ValueError("JSON numbers must be finite.")
        if isinstance(item, dict):
            for child in item.values():
                finite(child)
        elif isinstance(item, list):
            for child in item:
                finite(child)

    finite(value)
    return value


def braintrust_package(read, *, discard_legacy_models=False):
    """Pure structural validation. read(name) returns bytes; no seller execution.

    Shared by the ZIP checker and local helper. Errors contain field paths,
    never case values, prompt/code contents, or credentials.
    Only the buyer helper discards account settings from older packages; new
    submissions must omit them. Neither path changes source files.
    """

    def fail(path, message):
        raise ValueError(f"braintrust/{path}: {message}")

    def obj(value, allowed, required, path):
        if not isinstance(value, dict):
            fail(path, "must be an object.")
        if set(value) - set(allowed):
            fail(
                path, "contains unsupported fields; consult the allowed fields in Docs."
            )
        if set(required) - set(value):
            fail(
                path,
                "missing required fields: "
                + ", ".join(sorted(set(required) - set(value))),
            )
        return value

    def string(value, path):
        if not isinstance(value, str) or not value.strip():
            fail(path, "must be a nonempty string.")

    def strings(value, path):
        if not isinstance(value, list) or any(not isinstance(v, str) for v in value):
            fail(path, "must be an array of strings.")

    loaded = {}
    for name in ("README.md", "dataset.json", "tasks.json", "scorers.json"):
        try:
            raw = read(name)
            if not 0 < len(raw) <= BRAINTRUST_FILE_BYTES:
                fail(name, "must be a nonempty file up to 5 MiB.")
            text = raw.decode("utf-8-sig")
            if not text.strip():
                fail(name, "must be nonempty UTF-8 text.")
            loaded[name] = native_json(text) if name.endswith(".json") else text
        except (OSError, KeyError):
            if name == "tasks.json":
                fail(
                    name,
                    "required task setup is missing or unreadable. Ask the seller for a complete updated package; older compatibility checks did not require tasks.",
                )
            fail(name, "required file is missing or unreadable.")
        except (UnicodeError, json.JSONDecodeError, RecursionError):
            fail(name, "must contain valid UTF-8 text and JSON where applicable.")
        except ValueError as error:
            if str(error).startswith("braintrust/"):
                raise
            fail(
                name,
                "invalid JSON: duplicate keys or non-finite numbers are not allowed.",
            )
    cases, tasks, scorers = (
        loaded["dataset.json"],
        loaded["tasks.json"],
        loaded["scorers.json"],
    )
    for name, values in (
        ("dataset.json", cases),
        ("tasks.json", tasks),
        ("scorers.json", scorers),
    ):
        if not isinstance(values, list) or not values:
            fail(name, "must be a nonempty array.")
    ids = set()
    for index, case in enumerate(cases):
        path = f"dataset.json[{index}]"
        obj(case, {"input", "id", "expected", "metadata", "tags"}, {"input"}, path)
        if "id" in case:
            string(case["id"], path + ".id")
            if case["id"] in ids:
                fail(path + ".id", "duplicate supplied ID.")
            ids.add(case["id"])
        if "metadata" in case and not isinstance(case["metadata"], dict):
            fail(path + ".metadata", "must be an object.")
        if "tags" in case:
            strings(case["tags"], path + ".tags")
    slugs = set()
    functions = [
        (filename, index, kind, definition)
        for filename, kind, definitions in (
            ("tasks.json", "task", tasks),
            ("scorers.json", "scorer", scorers),
        )
        for index, definition in enumerate(definitions)
    ]
    for filename, index, kind, scorer in functions:
        path = f"{filename}[{index}]"
        obj(
            scorer,
            {
                "name",
                "slug",
                "description",
                "tags",
                "function_type",
                "function_data",
                "function_schema",
                "prompt_data",
            },
            {"name", "slug", "function_type", "function_data"},
            path,
        )
        string(scorer["name"], path + ".name")
        slug = scorer["slug"]
        if (
            not isinstance(slug, str)
            or not re.fullmatch(r"[a-z0-9]+(?:[-_][a-z0-9]+)*", slug)
            or len(slug) > 80
        ):
            fail(
                path + ".slug",
                "use 1–80 lowercase letters/digits with single hyphen/underscore separators.",
            )
        if slug in slugs:
            fail(
                path + ".slug",
                "duplicate task/scorer slug; slugs must be unique across both files.",
            )
        slugs.add(slug)
        if scorer["function_type"] != kind:
            fail(path + ".function_type", f"must be {kind}.")
        if "description" in scorer and not isinstance(scorer["description"], str):
            fail(path + ".description", "must be a string.")
        if "tags" in scorer:
            strings(scorer["tags"], path + ".tags")
        if "function_schema" in scorer:
            schema = obj(
                scorer["function_schema"],
                {"parameters", "returns"},
                set(),
                path + ".function_schema",
            )
            for value in schema.values():
                if not isinstance(value, (dict, bool)):
                    fail(
                        path + ".function_schema",
                        "parameters/returns must be JSON schema objects or booleans.",
                    )

            # Schemas are data; external references cannot be resolved by this importer.
            def refs(value, path=path):
                if isinstance(value, dict):
                    for key, child in value.items():
                        if key in {"$ref", "$dynamicRef"} and (
                            not isinstance(child, str) or not child.startswith("#")
                        ):
                            fail(
                                path + ".function_schema",
                                "external schema references are unsupported.",
                            )
                        refs(child)
                elif isinstance(value, list):
                    for child in value:
                        refs(child)

            refs(schema)
        data = obj(
            scorer["function_data"], {"type", "data"}, {"type"}, path + ".function_data"
        )
        if data["type"] == "code":
            if "prompt_data" in scorer:
                fail(path, "code functions cannot include prompt_data.")
            inline = obj(
                data.get("data"),
                {"type", "runtime_context", "code"},
                {"type", "runtime_context", "code"},
                path + ".function_data.data",
            )
            if inline["type"] != "inline":
                fail(
                    path + ".function_data.data.type", "only inline code is supported."
                )
            runtime = obj(
                inline["runtime_context"],
                {"runtime", "version"},
                {"runtime", "version"},
                path + ".runtime_context",
            )
            if (runtime["runtime"], runtime["version"]) not in (
                ("node", "22"),
                ("python", "3.12"),
            ):
                fail(
                    path + ".runtime_context",
                    "supported runtimes are node 22 and python 3.12.",
                )
            string(inline["code"], path + ".code")
        elif data["type"] == "prompt":
            if set(data) != {"type"}:
                fail(
                    path + ".function_data", "prompt function_data contains only type."
                )
            prompt = obj(
                scorer.get("prompt_data"),
                {"prompt", "options", "template_format"}
                | ({"parser"} if kind == "scorer" else set()),
                {"prompt", "options"} | ({"parser"} if kind == "scorer" else set()),
                path + ".prompt_data",
            )
            if "template_format" in prompt and prompt["template_format"] not in (
                "mustache",
                "nunjucks",
                "none",
            ):
                fail(path + ".template_format", "use mustache, nunjucks, or none.")
            if (
                kind == "task"
                and prompt.get("template_format", "mustache") != "mustache"
            ):
                fail(
                    path + ".template_format",
                    "prompt tasks use mustache with input-only bindings.",
                )
            chat = obj(
                prompt["prompt"],
                {"type", "messages"},
                {"type", "messages"},
                path + ".prompt",
            )
            if (
                chat["type"] != "chat"
                or not isinstance(chat["messages"], list)
                or not chat["messages"]
            ):
                fail(path + ".prompt", "must be chat with a nonempty messages array.")
            for i, message in enumerate(chat["messages"]):
                mp = path + f".prompt.messages[{i}]"
                obj(message, {"role", "content", "name"}, {"role", "content"}, mp)
                if message["role"] not in ("system", "developer", "user", "assistant"):
                    fail(
                        mp + ".role",
                        "use system, developer, user, or assistant; tools are unsupported.",
                    )
                string(message["content"], mp + ".content")
                if kind == "task":
                    # Scan once: unmatched seller-supplied braces must not cause
                    # a backtracking regex to repeatedly scan a multi-MiB prompt.
                    content, offset = message["content"], 0
                    while (start := content.find("{{", offset)) != -1:
                        width = 3 if content.startswith("{{{", start) else 2
                        end = content.find("}" * width, start + width)
                        if end == -1:
                            fail(mp + ".content", "malformed task input binding.")
                        binding = content[start + width : end].strip()
                        if not re.fullmatch(r"input(?:\.[A-Za-z0-9_-]+)*", binding):
                            fail(
                                mp + ".content",
                                "task bindings may reference only input or input.field; expected answers, outputs, and grading metadata are not task inputs.",
                            )
                        offset = end + width
                if "name" in message:
                    string(message["name"], mp + ".name")
            options = obj(
                prompt["options"],
                {"model", "params", "position", "endpoint_name"},
                set(),
                path + ".options",
            )
            for key in ("model", "endpoint_name"):
                if key in options:
                    if not discard_legacy_models:
                        fail(
                            path + ".options." + key,
                            "remove this field; the importer selects model/provider configuration from the buyer's account.",
                        )
                    # Older published files stay byte-for-byte intact. Their
                    # account-specific settings are never used or displayed.
                    options.pop(key)
            if "position" in options:
                string(options["position"], path + ".options.position")
            if "params" in options:
                params = obj(
                    options["params"],
                    NUMERIC_PARAMS
                    | BOOL_PARAMS
                    | {"stop", "stop_sequences", "reasoning_effort", "verbosity"},
                    set(),
                    path + ".options.params",
                )
                for key, value in params.items():
                    pp = path + ".options.params." + key
                    if key in NUMERIC_PARAMS and type(value) not in (int, float):
                        fail(pp, "must be a finite number.")
                    if key in BOOL_PARAMS and not isinstance(value, bool):
                        fail(pp, "must be boolean.")
                    if key in ("stop", "stop_sequences"):
                        strings(value, pp)
                    if key == "reasoning_effort" and value not in (
                        "none",
                        "minimal",
                        "low",
                        "medium",
                        "high",
                    ):
                        fail(pp, "use none, minimal, low, medium, or high.")
                    if key == "verbosity" and value not in ("low", "medium", "high"):
                        fail(pp, "use low, medium, or high.")
            if kind == "task":
                continue
            parser = obj(
                prompt["parser"],
                {"type", "use_cot", "choice_scores", "allow_skip"},
                {"type", "use_cot", "choice_scores"},
                path + ".parser",
            )
            if parser["type"] != "llm_classifier":
                fail(path + ".parser.type", "must be llm_classifier.")
            for key in ("use_cot", "allow_skip"):
                if key in parser and not isinstance(parser[key], bool):
                    fail(path + ".parser." + key, "must be boolean.")
            choices = parser["choice_scores"]
            if (
                not isinstance(choices, dict)
                or not choices
                or any(
                    not key.strip()
                    or type(value) not in (int, float)
                    or not 0 <= value <= 1
                    for key, value in choices.items()
                )
            ):
                fail(
                    path + ".parser.choice_scores",
                    "use a nonempty choice-to-score mapping with finite numbers from 0 to 1.",
                )
        else:
            fail(
                path + ".function_data.type",
                "only code and prompt tasks/scorers are supported.",
            )
    return cases, tasks, scorers


class NoCredentialRedirect(HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        raise ValueError("Braintrust redirect refused; credentials were not forwarded.")


class BraintrustClient:
    def __init__(self, key):
        self.key = key
        self.opener = build_opener(NoCredentialRedirect())

    def request(
        self, method, path, body=None, *, _start_only=False, _model_scope=None, **query
    ):
        if not path.startswith("/v1/") or "?" in path or "#" in path:
            raise ValueError("Unsupported Braintrust API path.")
        headers = {
            "Authorization": "Bearer " + self.key,
            "Content-Type": "application/json",
            "Accept": "text/event-stream" if _start_only else "application/json",
        }
        if _model_scope is not None:
            project_id, provider_name = _model_scope
            if (
                method != "GET"
                or path != "/v1/proxy/models"
                or not model_label(provider_name)
            ):
                raise ValueError("Invalid Braintrust model discovery scope.")
            headers.update(
                {
                    "x-bt-project-id": str(UUID(project_id)),
                    "x-bt-endpoint-name": provider_name,
                }
            )
        url = BRAINTRUST_API + path + ("?" + urlencode(query) if query else "")
        request = Request(
            url,
            method=method,
            headers=headers,
            data=json.dumps(body, allow_nan=False).encode()
            if body is not None
            else None,
        )
        try:
            with self.opener.open(request, timeout=60) as response:
                if _start_only:
                    return read_experiment_start(response, body["project_id"])
                return native_json(response.read().decode("utf-8"))
        except HTTPError as error:
            # Never echo provider bodies/URLs; these can include private data or credentials.
            detail = {
                401: "invalid API key",
                403: "insufficient permissions",
                429: "rate limited",
            }.get(error.code, "request rejected")
            raise ValueError(
                f"Braintrust HTTP {error.code}: {detail}. No write was retried."
            ) from None
        except (URLError, OSError, UnicodeError, ValueError) as error:
            if isinstance(error, ValueError) and str(error).startswith(
                "Braintrust redirect"
            ):
                raise
            raise ValueError(
                "Braintrust transport/read response failed; the outcome may be uncertain. No write was retried."
            ) from None

    def objects(self, kind, **filters):
        cursor, seen = None, set()
        while True:
            page = self.request(
                "GET",
                "/v1/" + kind,
                **filters,
                limit=100,
                **({"starting_after": cursor} if cursor else {}),
            )
            records = page.get("objects") if isinstance(page, dict) else None
            if not isinstance(records, list):
                raise ValueError("Braintrust returned an invalid object list.")
            if not records:
                return
            for record in records:
                identity = resource_id(record)
                if identity in seen:
                    raise ValueError(
                        "Braintrust pagination repeated an object; stopped safely."
                    )
                seen.add(identity)
                yield record
            cursor = resource_id(records[-1])


def read_experiment_start(response, project_id):
    """Read only the SSE start acknowledgement, never results or completion."""
    event, data, size = "", [], 0
    deadline = monotonic() + 60
    while monotonic() < deadline:
        line = response.readline(65537)
        size += len(line)
        if not line or size > 65536:
            break
        line = line.decode("utf-8").rstrip("\r\n")
        if not line:
            if event == "start" and data:
                started = native_json("\n".join(data))
                if (
                    not isinstance(started, dict)
                    or started.get("projectId") != project_id
                ):
                    break
                identity = resource_id({"id": started.get("experimentId")})
                url = started.get("experimentUrl")
                if not isinstance(url, str) or any(
                    c.isspace() or ord(c) < 32 or ord(c) == 127 for c in url
                ):
                    break
                parsed = urlsplit(url)
                if (
                    parsed.scheme != "https"
                    or parsed.netloc != "www.braintrust.dev"
                    or not parsed.path.startswith("/app/")
                    or "/experiments/" not in parsed.path
                ):
                    break
                return identity, url
            if event or data:
                # Do not parse error bodies or a result event as a successful start.
                break
            continue  # SSE heartbeat/comment-only frame
        field, separator, value = line.partition(":")
        if separator and value.startswith(" "):
            value = value[1:]
        if field == "event":
            event = value
        elif field == "data":
            data.append(value)
    raise ValueError("No valid Braintrust experiment start acknowledgement.")


class ExperimentStartError(ValueError):
    """An optional run failed after a successful, verified import."""


def offer_experiment(client, project_id, dataset_id, tasks, scorers, case_count):
    print("\nStart an experiment")
    print(
        f"  Runs one imported task on all {case_count} cases with the imported scorers."
    )
    print(
        "  Uses your Braintrust model/provider access; model calls may incur charges."
    )
    print("  Seller code runs with Braintrust hosted-function permissions.")
    try:
        answer = input("Would you like to start an experiment? [y/N]: ").strip().lower()
        if answer not in {"y", "yes"}:
            return False
        selected = tasks[0]
        if len(tasks) > 1:
            choices = {str(i): task for i, task in enumerate(tasks, 1)}
            while True:
                print("\nChoose a task:")
                for number, (task, _) in choices.items():
                    print(f"  {number}. {task['name']!r}")
                choice = input(f"Task number [1-{len(tasks)}, q to cancel]: ").strip()
                if choice.lower() == "q":
                    return False
                selected = choices.get(choice.lstrip("0"))
                if selected is not None:
                    break
                print(f"Enter a task number from 1 to {len(tasks)}, or q to cancel.")
    except (EOFError, KeyboardInterrupt):
        print()
        return False
    task, task_id = selected
    name = "EvalMarket experiment " + uuid4().hex
    print(f"\nStarting {name!r}", flush=True)
    print(f"  Task: {task['name']!r}", flush=True)
    try:
        identity, url = client.request(
            "POST",
            "/v1/eval",
            {
                "project_id": project_id,
                "data": {"dataset_id": dataset_id},
                "task": {"function_id": task_id},
                "scores": [{"function_id": identity} for _, identity in scorers],
                "experiment_name": name,
                "stream": True,
                "max_concurrency": 1,
                "is_public": False,
            },
            _start_only=True,
        )
    except (ValueError, OSError, KeyboardInterrupt, EOFError):
        raise ExperimentStartError(
            f"Experiment start unconfirmed. Inspect project {project_id} for {name!r} before starting another run; it may already be running. No request was retried. Your imported resources are complete and do not need to be imported again."
        ) from None
    print(f"\nExperiment started. ID: {identity}")
    print(f"  {url}")
    print("Braintrust will continue the run. Open the link for progress and results.")
    return True


def resource_id(record):
    try:
        return str(UUID(record["id"]))
    except (KeyError, TypeError, ValueError, AttributeError):
        raise ValueError(
            "Braintrust returned no valid resource ID; inspect the project before retrying."
        ) from None


def same_json(left, right):
    """JSON-aware equality distinguishes booleans from numbers at every depth."""
    if isinstance(left, dict):
        return (
            isinstance(right, dict)
            and left.keys() == right.keys()
            and all(same_json(v, right[k]) for k, v in left.items())
        )
    if isinstance(left, list):
        return (
            isinstance(right, list)
            and len(left) == len(right)
            and all(same_json(a, b) for a, b in zip(left, right))
        )
    if type(left) in (int, float):
        return type(right) in (int, float) and left == right
    return type(left) is type(right) and left == right


def verify_fields(expected, actual, path="resource"):
    """Supplied values must survive; provider-added optional null fields may appear."""
    if isinstance(expected, dict) and isinstance(actual, dict):
        if path not in {"resource", "dataset", "task", "scorer"}:
            for key in actual.keys() - expected.keys():
                if actual[key] is not None and not (
                    path.endswith(".function_data.data") and key == "code_hash"
                ):
                    raise ValueError(
                        f"Readback mismatch at {path}; unexpected non-null fields."
                    )
        for key, value in expected.items():
            if key not in actual:
                raise ValueError(f"Readback mismatch at {path}.{key} (missing field).")
            verify_fields(value, actual[key], path + "." + key)
    elif not same_json(expected, actual):
        raise ValueError(f"Readback mismatch at {path}; import is not verified.")


def model_label(value):
    return (
        isinstance(value, str)
        and 0 < len(value) <= 512
        and value.strip() == value
        and value.isprintable()
    )


def braintrust_model_choices(client, project):
    """Use visible configuration first; catalogs are incomplete, not access probes."""
    choices, incomplete = {}, False
    try:
        org_id = resource_id({"id": project.get("org_id")})
        provider_records = list(client.objects("ai_secret"))
    except (ValueError, OSError):
        return [], True
    for record in provider_records:
        if record.get("org_id") != org_id:
            continue
        provider = record.get("name")
        metadata = record.get("metadata") or {}
        if not model_label(provider) or not isinstance(metadata, dict):
            incomplete = True
            continue
        custom = metadata.get("customModels") or {}
        configured = metadata.get("models") or []
        excluded = metadata.get("excludeDefaultModels")
        if (
            not isinstance(custom, dict)
            or not isinstance(configured, list)
            or (excluded is not None and type(excluded) is not bool)
        ):
            incomplete = True
            continue
        # Custom model flavor is authoritative; do not offer embedding/realtime
        # definitions as answering/judge models, even if listed in models too.
        non_chat = {
            name
            for name, spec in custom.items()
            if not isinstance(spec, dict) or spec.get("flavor") != "chat"
        }
        for name in custom:
            if model_label(name) and name not in non_chat:
                choices[name, provider] = "configured"
        if excluded is not True:
            for name in configured:
                if model_label(name) and name not in non_chat:
                    choices[name, provider] = "configured"
                elif not model_label(name):
                    incomplete = True
        if configured or excluded is True:
            continue
        # Only unrestricted providers need an upstream catalog. No seller endpoint
        # is followed, and no provider credential or inference request is needed.
        try:
            catalog = client.request(
                "GET", "/v1/proxy/models", _model_scope=(resource_id(project), provider)
            )
            entries = catalog.get("data") if isinstance(catalog, dict) else None
            if not isinstance(entries, list):
                raise ValueError("Invalid model catalog.")
            for entry in entries:
                if not isinstance(entry, dict):
                    incomplete = True
                    continue
                name = entry.get("id")
                if (
                    model_label(name)
                    and name not in non_chat
                    and entry.get("flavor", "chat") == "chat"
                ):
                    choices.setdefault((name, provider), "provider catalog")
            if not entries or catalog.get("has_more") or catalog.get("next_page_token"):
                incomplete = True
        except (ValueError, OSError):
            incomplete = True
    return [
        (model, provider, source)
        for (model, provider), source in sorted(
            choices.items(),
            key=lambda item: (item[0][0].casefold(), item[0][1].casefold()),
        )
    ], incomplete


def choose_braintrust_models(client, project, functions, interactive):
    pending = [f for f in functions if f["function_data"]["type"] == "prompt"]
    if not pending:
        return
    print("\nChoose models")
    print(
        "  Select a model/provider from your account for each prompt task and LLM scorer."
    )
    print("  Discovery does not run models. Prompts and parameters stay unchanged.")
    try:
        choices, incomplete = braintrust_model_choices(client, project)
    except KeyboardInterrupt:
        raise ValueError("Model selection cancelled; no writes were made.") from None
    print(
        "  Choices come from visible organization configuration or provider catalogs."
    )
    print(
        "  Project-only providers/overrides may be missing; chat support and access are not tested."
    )
    if incomplete:
        print(
            "  Model discovery is incomplete; only discovered choices can be selected."
        )
    if not choices:
        raise ValueError(
            "No model/provider choices could be discovered for this account/project. Check your Braintrust provider configuration and API-key access, then rerun. No writes were made."
        )
    try:
        for definition in pending:
            options = definition["prompt_data"]["options"]
            print(f"\n{definition['function_type'].title()}: {definition['name']!r}")
            # Only buyer CLI values can be present: package settings were discarded.
            candidates = [
                pair
                for pair in choices
                if ("model" not in options or pair[0] == options["model"])
                and (
                    "endpoint_name" not in options
                    or pair[1] == options["endpoint_name"]
                )
            ]
            if not candidates:
                raise ValueError(
                    f"No discovered model/provider matches the buyer CLI selection for {definition['slug']!r}. Check --model/--provider or rerun without those flags to choose from your account. No writes were made."
                )
            if "model" in options and len(candidates) == 1:
                model, provider, _ = candidates[0]
                options.update(model=model, endpoint_name=provider)
                print(f"  Selected: {model!r} via {provider!r}")
                continue
            if not interactive:
                raise ValueError(
                    f"A unique account model/provider selection is required for {definition['slug']!r}. Supply --model and, if ambiguous, --provider for each prompt task/LLM scorer, or run interactively. No writes were made."
                )
            offset = 0
            while True:
                for index, (model, provider, source) in enumerate(
                    candidates[offset : offset + 20], offset + 1
                ):
                    print(f"  {index}. {model!r} via {provider!r} ({source})")
                if len(candidates) > 20:
                    print(
                        f"  Showing {offset + 1}-{min(offset + 20, len(candidates))} of {len(candidates)}; n = next, p = previous."
                    )
                choice = input("Model choice [number required, q cancel]: ").strip()
                if choice.lower() == "q":
                    raise ValueError("Import cancelled; no writes were made.")
                if choice.lower() in {"n", "p"} and len(candidates) > 20:
                    offset = (
                        min(offset + 20, ((len(candidates) - 1) // 20) * 20)
                        if choice.lower() == "n"
                        else max(0, offset - 20)
                    )
                    continue
                selected = {
                    str(index): pair for index, pair in enumerate(candidates, 1)
                }.get(choice.lstrip("0"))
                if selected is None:
                    print(
                        "Choose a listed model number or q to cancel; a model selection is required."
                    )
                    continue
                model, provider, _ = selected
                options.update(model=model, endpoint_name=provider)
                break
    except (EOFError, KeyboardInterrupt):
        raise ValueError("Model selection cancelled; no writes were made.") from None


def import_braintrust(folder, project_id=None, yes=False, models=None, providers=None):
    folder = Path(folder).resolve()

    def read(name):
        path = folder / name
        if (
            path.is_symlink()
            or not path.is_file()
            or not 0 < path.stat().st_size <= BRAINTRUST_FILE_BYTES
        ):
            raise ValueError(
                f"braintrust/{name}: required regular file must be nonempty and up to 5 MiB."
                + (
                    " Ask the seller for a complete updated package with task setup; older compatibility checks did not require tasks."
                    if name == "tasks.json"
                    else ""
                )
            )
        return path.read_bytes()

    cases, tasks, scorers = braintrust_package(read, discard_legacy_models=True)
    functions = tasks + scorers
    interactive = sys.stdin.isatty()
    if not interactive and (not project_id or not yes):
        raise ValueError(
            "Noninteractive Braintrust imports require --project-id and --yes. Nothing was uploaded."
        )
    if project_id:
        try:
            project_id = str(UUID(project_id))
        except ValueError:
            raise ValueError(
                "--project-id must be a Braintrust project UUID."
            ) from None
    for flag, supplied, field in (
        ("--model", models, "model"),
        ("--provider", providers, "endpoint_name"),
    ):
        overrides = {}
        for value in supplied or []:
            slug, separator, label = value.partition("=")
            if (
                not separator
                or not model_label(label)
                or slug in overrides
                or not any(
                    scorer["slug"] == slug
                    and scorer["function_data"]["type"] == "prompt"
                    for scorer in functions
                )
            ):
                raise ValueError(
                    f"{flag} requires a unique prompt task or LLM scorer slug=nonempty-name override."
                )
            overrides[slug] = label
        for scorer in functions:
            if scorer["slug"] in overrides:
                scorer["prompt_data"]["options"][field] = overrides[scorer["slug"]]
    key = os.environ.get("BRAINTRUST_API_KEY")
    if not key:
        if not interactive:
            raise ValueError(
                "Set BRAINTRUST_API_KEY locally, or use an interactive terminal for a masked prompt."
            )
        with warnings.catch_warnings():
            warnings.simplefilter("error", getpass.GetPassWarning)
            try:
                key = getpass.getpass("Braintrust API key (hidden): ")
            except (getpass.GetPassWarning, EOFError):
                raise ValueError(
                    "A masked terminal prompt is unavailable. Set BRAINTRUST_API_KEY locally."
                ) from None
    if not key or key.strip() != key or any(c.isspace() or ord(c) < 32 for c in key):
        raise ValueError(
            "A nonempty Braintrust API key without whitespace is required."
        )
    client = BraintrustClient(key)
    if project_id:
        project = client.request("GET", "/v1/project/" + project_id)
        if resource_id(project) != project_id:
            raise ValueError("The accessible project did not match --project-id.")
    else:
        projects = list(client.objects("project"))
        if not projects:
            raise ValueError(
                "No accessible Braintrust projects. Create/configure one in Braintrust first."
            )
        project_numbers = {str(i): project for i, project in enumerate(projects, 1)}
        while True:
            print("\nAvailable Braintrust projects (choose the number on the left):")
            for number, project in project_numbers.items():
                print(
                    f"  {number}. {project.get('name', 'Unnamed project')!r} ({resource_id(project)})"
                )
            try:
                choice = input(
                    f"Destination project number [1-{len(projects)}, q to cancel]: "
                ).strip()
            except EOFError:
                raise ValueError(
                    "No project selected; no writes were made. Rerun in an interactive terminal or supply --project-id and --yes."
                ) from None
            if choice.lower() == "q":
                raise ValueError("Import cancelled; no writes were made.")
            project = project_numbers.get(choice.lstrip("0"))
            if project is not None:
                break
            print(f"Enter a project number from 1 to {len(projects)}, or q to cancel.")
        project_id = resource_id(project)
    choose_braintrust_models(client, project, functions, interactive)
    token = uuid4().hex
    name = "EvalMarket import " + token
    print("\nBraintrust import")
    print(f"  Destination: {project.get('name', 'Unnamed project')!r} ({project_id})")
    print(f"  Dataset: {len(cases)} cases")
    print(f"  Import ID: {token}")
    prepared = []
    for kind, scorer in [("task", task) for task in tasks] + [
        ("scorer", scorer) for scorer in scorers
    ]:
        native = copy.deepcopy(scorer)
        native.update(
            project_id=project_id,
            name=scorer["name"] + " [" + token + "]",
            slug=scorer["slug"] + "-" + token,
        )
        # The package role is "task", but Braintrust's Prompt picker lists
        # answering prompts as "llm". "task" is callable via the API but hidden
        # from that picker. Keep the seller's prompt and grading logic intact.
        if kind == "task" and native["function_data"]["type"] == "prompt":
            native["function_type"] = "llm"
        prepared.append((kind, native))
        print(f"  {kind.title()}: {scorer['name']!r}")
        if native["function_data"]["type"] == "prompt":
            print(f"    Model: {native['prompt_data']['options']['model']!r}")
            print(
                f"    Provider: {native['prompt_data']['options'].get('endpoint_name', 'Braintrust routing default')!r}"
            )
    print(
        "\nImport installs the evaluation without running it. Model changes can change outputs and judgments."
    )
    if (
        not yes
        and input("Create this dataset, task set, and scorer set? [y/N]: ")
        .strip()
        .lower()
        != "y"
    ):
        raise ValueError("Import cancelled; no writes were made.")
    # Check all identities before any write. No automatic retry or fallback name.
    if list(client.objects("dataset", project_id=project_id, dataset_name=name)):
        raise ValueError("Dataset name collision; no writes were made.")
    for _, native in prepared:
        if list(client.objects("function", project_id=project_id, slug=native["slug"])):
            raise ValueError("Task/scorer slug collision; no writes were made.")
    rows = []
    row_ids = {case["id"] for case in cases if "id" in case}
    for case in cases:
        row = copy.deepcopy(case)
        if "id" not in row:
            identity = str(uuid4())
            while identity in row_ids:
                identity = str(uuid4())
            row["id"] = identity
            row_ids.add(identity)
        rows.append(row)
    confirmed, stage = [], "create dataset (outcome uncertain if no response)"
    saved_tasks, saved_scorers = [], []
    print("\nUploading and verifying...", flush=True)
    try:
        dataset_body = {
            "project_id": project_id,
            "name": name,
            "description": "Seller-native EvalMarket import " + token,
            "metadata": {"evalmarket_import_id": token},
        }
        dataset = client.request("POST", "/v1/dataset", dataset_body)
        dataset_id = resource_id(dataset)
        # The API reuses names. A foreign/missing marker must stop before insert.
        verify_fields(dataset_body, dataset, "dataset")
        confirmed.append("dataset created: " + dataset_id)
        for start in range(0, len(rows), 100):
            batch = rows[start : start + 100]
            stage = f"upload dataset rows {start + 1}–{start + len(batch)} (outcome uncertain if interrupted)"
            response = client.request(
                "POST", f"/v1/dataset/{dataset_id}/insert", {"events": batch}
            )
            if not isinstance(response, dict) or response.get("row_ids") != [
                row["id"] for row in batch
            ]:
                raise ValueError(
                    "Dataset insert did not acknowledge the exact supplied row IDs."
                )
            confirmed.append(f"rows {start + 1}–{start + len(batch)} acknowledged")
        stage = "dataset readback"
        fetched, cursor, cursors = {}, None, set()
        while True:
            page = client.request(
                "GET",
                f"/v1/dataset/{dataset_id}/fetch",
                limit=100,
                **({"cursor": cursor} if cursor else {}),
            )
            if not isinstance(page, dict) or not isinstance(page.get("events"), list):
                raise ValueError("Invalid dataset readback response.")
            for row in page["events"]:
                if not isinstance(row, dict) or not isinstance(row.get("id"), str):
                    raise ValueError("Invalid dataset readback row.")
                # Fetch is newest-first; older versions can appear on later pages.
                fetched.setdefault(row["id"], row)
            cursor = page.get("cursor")
            if not cursor:
                break
            if not isinstance(cursor, str) or cursor in cursors:
                raise ValueError("Dataset readback cursor repeated or was invalid.")
            cursors.add(cursor)
        if fetched.keys() != row_ids:
            raise ValueError("Dataset readback IDs differ from the uploaded case set.")
        for row in rows:
            actual = fetched[row["id"]]
            for field in ("id", "input", "expected", "metadata", "tags"):
                if field in row:
                    if field not in actual or not same_json(row[field], actual[field]):
                        raise ValueError(
                            f"Dataset readback mismatch in {field}; supplied values were not preserved."
                        )
                elif actual.get(field) is not None:
                    raise ValueError(
                        f"Dataset readback introduced an unexpected {field} value."
                    )
        confirmed.append("all dataset rows verified")
        print(f"  Dataset verified: {len(rows)} cases", flush=True)
        for kind, native in prepared:
            stage = (
                "create "
                + kind
                + " "
                + native["slug"]
                + " (outcome uncertain if no response)"
            )
            created = client.request("POST", "/v1/function", native)
            function_id = resource_id(created)
            confirmed.append(
                kind + " created: " + function_id + " (" + native["slug"] + ")"
            )
            stage = "readback " + kind + " " + function_id
            verify_fields(
                native, client.request("GET", "/v1/function/" + function_id), kind
            )
            confirmed.append(kind + " verified: " + function_id)
            (saved_tasks if kind == "task" else saved_scorers).append(
                (native, function_id)
            )
            print(f"  {kind.title()} verified", flush=True)
    except (ValueError, OSError, KeyboardInterrupt, EOFError):
        print("\nImport incomplete. Stopped during: " + stage, file=sys.stderr)
        print(
            "Confirmed: " + ("; ".join(confirmed) or "no resources confirmed"),
            file=sys.stderr,
        )
        print(
            f"Inspect project {project_id}, dataset {name!r}, and functions with import suffix {token} before retrying. An interrupted write may have succeeded. Nothing was retried or deleted; a new invocation creates a new import.",
            file=sys.stderr,
        )
        raise
    print("\nImport complete. Dataset, tasks, and scorers are readback-verified.")
    print(f"  Dataset: {name!r}\n    ID: {dataset_id}")
    for kind, resources in (("Task", saved_tasks), ("Scorer", saved_scorers)):
        for native, identity in resources:
            print(f"  {kind}: {native['name']!r}\n    ID: {identity}")
    print("Tasks receive case input; expected answers stay with scorers.")
    if interactive and offer_experiment(
        client, project_id, dataset_id, saved_tasks, saved_scorers, len(cases)
    ):
        return dataset_id
    print("\nNo evaluation was run. Your imported resources are ready.")
    print(
        f"Open https://www.braintrust.dev and project {project.get('name', project_id)!r}."
    )
    if any(task["function_data"]["type"] == "prompt" for task in tasks):
        print(
            "  Experiments > + Experiment > + Task > Prompt: choose the imported prompt, dataset and scorers. Refresh if needed."
        )
    if any(task["function_data"]["type"] == "code" for task in tasks):
        print(
            "  Inline code tasks: use their saved IDs with the API/SDK. The UI task picker does not list them."
        )
    print(
        "Configure model/provider access before running; missing access is a setup failure. Seller code runs with Braintrust hosted-function permissions."
    )
    return dataset_id


def choose_model(run):
    records = run(["api", "playground-settings"])
    if not isinstance(records, list):
        raise ValueError("Could not read saved model configurations from LangSmith.")
    models = [
        record
        for record in records
        if isinstance(record, dict)
        and isinstance(record.get("id"), str)
        and record["id"]
        and record.get("available_in_evaluators", True) is True
    ]
    if not models:
        raise ValueError(
            "No saved models are available for evaluators in this LangSmith workspace. "
            "Add or enable one in Settings > Model configurations, then rerun. "
            "Nothing has been uploaded."
        )
    print("\nAvailable LangSmith judge models:")
    for index, model in enumerate(models, 1):
        # Only display the label and identity, never settings or credentials.
        print(f"  {index}. {model.get('name') or 'Unnamed model'!r} ({model['id']!r})")
    while True:
        try:
            choice = input(
                "Which model would you like to configure as your judge? "
                f"[1-{len(models)}, q to cancel]: "
            ).strip()
        except EOFError:
            raise ValueError(
                "Choose a model in an interactive terminal, or pass --model-config "
                "with an existing native model file. Nothing has been uploaded."
            ) from None
        if choice.lower() == "q":
            raise ValueError("Import cancelled. Nothing has been uploaded.")
        if choice.isascii() and choice.isdigit() and 1 <= int(choice) <= len(models):
            selected = models[int(choice) - 1]
            if (
                not isinstance(selected.get("settings"), dict)
                or not selected["settings"]
            ):
                raise ValueError(
                    "The selected model has no saved configuration. Nothing has been uploaded."
                )
            return {
                "playground_settings_id": selected["id"],
                "model": selected["settings"],
            }
        print(f"Enter a number from 1 to {len(models)}, or q to cancel.")


def langsmith_task(manifest, cases):
    """Check a native prompt without deserializing or executing seller objects."""

    def fail(message):
        raise ValueError(f"langchain/task.json: {message}")

    prompt_paths = {
        (namespace, "prompts", module, name)
        for namespace in ("langchain", "langchain_core")
        for module, name in (
            ("prompt", "PromptTemplate"),
            ("chat", "ChatPromptTemplate"),
        )
    } | {("langchain_core", "prompts", "structured", "StructuredPrompt")}
    identity = manifest.get("id") if isinstance(manifest, dict) else None
    if (
        not isinstance(identity, list)
        or len(identity) != 4
        or not all(isinstance(part, str) for part in identity)
        or tuple(identity) not in prompt_paths
        or manifest.get("lc") != 1
        or manifest.get("type") != "constructor"
        or not isinstance(manifest.get("kwargs"), dict)
    ):
        fail(
            "supply a native serialized PromptTemplate, ChatPromptTemplate, or StructuredPrompt without a model or runnable code."
        )
    options = manifest["kwargs"]
    variables = options.get("input_variables")
    if (
        not isinstance(variables, list)
        or len(variables) > 15
        or any(not isinstance(v, str) or not v.strip() for v in variables)
        or len(set(variables)) != len(variables)
    ):
        fail("input_variables must be an array of up to 15 unique nonempty names.")
    if not isinstance(cases, list) or not cases:
        fail("dataset.json must contain examples for the answering task.")
    for index, case in enumerate(cases, 1):
        inputs = case.get("inputs") if isinstance(case, dict) else None
        if not isinstance(inputs, dict) or any(v not in inputs for v in variables):
            fail(
                f"declared task variables must come from every example's inputs (example {index}); reference outputs are not task inputs."
            )
    if identity[-1] == "PromptTemplate":
        if (
            not isinstance(options.get("template"), str)
            or not options["template"].strip()
        ):
            fail("PromptTemplate needs a nonempty template.")
    elif not isinstance(options.get("messages"), list) or not options["messages"]:
        fail("chat/structured prompts need a nonempty messages array.")
    if identity[-1] == "StructuredPrompt" and (
        not isinstance(options.get("schema_"), dict) or not options["schema_"]
    ):
        fail("StructuredPrompt needs its nonempty native schema_ object.")

    def check_objects(value):
        if isinstance(value, dict):
            if value.get("lc") == 1:
                path = value.get("id")
                if (
                    value.get("type") != "constructor"
                    or not isinstance(path, list)
                    or not all(isinstance(p, str) for p in path)
                    or len(path) < 3
                    or path[0] not in ("langchain", "langchain_core")
                    or not (
                        path[1] in ("prompts", "messages")
                        or path[1:3] == ["schema", "messages"]
                    )
                ):
                    fail(
                        "only native prompt/message objects are supported; omit models, secrets, and executable runnables."
                    )
            for child in value.values():
                check_objects(child)
        elif isinstance(value, list):
            for child in value:
                check_objects(child)

    check_objects(manifest)
    return manifest


def langsmith_id(value, label):
    try:
        return str(UUID(value))
    except (ValueError, TypeError, AttributeError):
        raise ValueError(
            f"LangSmith returned no valid {label} ID; inspect your workspace before retrying."
        ) from None


def import_langsmith_task(run, manifest, dataset_id, auth):
    """Save a fresh private native prompt and verify it before offering the UI."""
    handle = "evalmarket-task-" + uuid4().hex
    print(f"Importing private answering prompt: {handle}", flush=True)
    response = run(
        ["api", "repos", "--method", "POST", "--input", "-"],
        {"repo_handle": handle, "is_public": False, "repo_type": "prompt"},
    )
    repo = response.get("repo") if isinstance(response, dict) else None
    if (
        not isinstance(repo, dict)
        or repo.get("repo_handle") != handle
        or repo.get("is_public") is not False
    ):
        raise ValueError(
            f"Could not confirm the new private prompt {handle}; inspect your workspace before retrying."
        )
    workspace = langsmith_id(repo.get("tenant_id"), "prompt workspace")
    if auth.get("workspace_id") and workspace != langsmith_id(
        auth["workspace_id"], "current workspace"
    ):
        raise ValueError(
            "The saved prompt belongs to a different workspace. Import stopped."
        )
    path = f"commits/-/{handle}"
    response = run(
        ["api", path, "--method", "POST", "--input", "-"],
        {"manifest": manifest, "skip_webhooks": True},
    )
    commit = response.get("commit") if isinstance(response, dict) else None
    commit_hash = commit.get("commit_hash") if isinstance(commit, dict) else None
    if not isinstance(commit_hash, str) or not re.fullmatch(
        r"[a-fA-F0-9]{8,64}", commit_hash
    ):
        raise ValueError(
            f"No confirmed commit for prompt {handle}; inspect it before retrying."
        )
    saved = run(["api", f"{path}/{commit_hash}"])
    if (
        not isinstance(saved, dict)
        or saved.get("commit_hash") != commit_hash
        or saved.get("manifest") != manifest
    ):
        raise ValueError(
            f"Prompt {handle} readback did not match task.json. No Playground was opened."
        )
    print(f"Verified answering prompt: {handle} ({commit_hash})", flush=True)
    origins = {
        "https://api.smith.langchain.com": "https://smith.langchain.com",
        "https://eu.api.smith.langchain.com": "https://eu.smith.langchain.com",
    }
    endpoint = str(auth.get("api_url", "")).rstrip("/")
    if endpoint.endswith("/api/v1"):
        endpoint = endpoint[:-7]
    origin = origins.get(endpoint)
    if not origin:
        print(
            "Your CLI endpoint has no known Playground browser origin. Open your LangSmith site, select the imported prompt and dataset, enable the evaluator, and review the model before clicking Start."
        )
        return None
    return f"{origin}/o/{workspace}/prompts/{handle}/playground?" + urlencode(
        {"commit": commit_hash, "datasetId": dataset_id}
    )


class PlaygroundSetupError(ValueError):
    """The import succeeded, but preparing the optional browser handoff failed."""


def offer_langsmith_playground(run, url, dataset_id, rule_id):
    print(f"\nLangSmith Playground: {url}")
    print(
        "Review/select your answering model and click Start in LangSmith to run the experiment."
    )
    if not sys.stdin.isatty():
        print(
            "Unattended import: Playground not opened; enable the imported evaluator in LangSmith when ready."
        )
        return
    print(
        "Opening the Playground enables the imported evaluator for all cases in this dataset. Model calls after Start may incur charges."
    )
    try:
        answer = (
            input(
                "Would you like to start an experiment in the LangSmith Playground? [y/N]: "
            )
            .strip()
            .lower()
        )
    except (EOFError, KeyboardInterrupt):
        print()
        answer = ""
    if answer not in {"y", "yes"}:
        print(
            "Playground skipped. Sampling remains 0; enable the imported evaluator when you are ready to run."
        )
        return
    print(f"Preparing evaluator rule {rule_id} for dataset {dataset_id}...", flush=True)
    try:
        # Verify attachment before updating only the newly imported rule.
        path = f"runs/rules/{rule_id}"

        def read_rule():
            # The native /{rule_id} endpoint has PATCH/DELETE, but no GET.
            response = run(
                [
                    "api",
                    "/api/v2/runs/rules?"
                    + urlencode({"id": rule_id, "dataset_id": dataset_id}),
                ]
            )
            records = response.get("items") if isinstance(response, dict) else None
            return (
                records[0]
                if isinstance(records, list)
                and len(records) == 1
                and response.get("next_cursor") is None
                else None
            )

        rule = read_rule()
        if (
            not isinstance(rule, dict)
            or rule.get("id") != rule_id
            or rule.get("dataset_id") != dataset_id
            or not isinstance(rule.get("display_name"), str)
            or not rule["display_name"].strip()
        ):
            raise ValueError(
                "The imported evaluator's dataset attachment could not be confirmed."
            )
        evaluator_id = langsmith_id(rule.get("evaluator_id"), "evaluator")
        run(
            ["api", path, "--method", "PATCH", "--input", "-"],
            {
                "display_name": rule["display_name"],
                "dataset_id": dataset_id,
                "evaluator_id": evaluator_id,
                "sampling_rate": 1,
                "is_enabled": True,
            },
        )
        rule = read_rule()
        if (
            not isinstance(rule, dict)
            or rule.get("id") != rule_id
            or rule.get("dataset_id") != dataset_id
            or rule.get("evaluator_id") != evaluator_id
            or rule.get("sampling_rate") != 1
            or rule.get("is_enabled") is not True
        ):
            raise ValueError("Evaluator preparation could not be confirmed.")
    except (
        ValueError,
        OSError,
        subprocess.CalledProcessError,
        EOFError,
        KeyboardInterrupt,
    ) as error:
        raise PlaygroundSetupError(
            f"Dataset {dataset_id}, evaluator rule {rule_id}, and answering prompt remain imported. Evaluator preparation was not confirmed and may already have changed sampling; inspect it in LangSmith. No request was retried. No experiment was started. Use {url} after checking the evaluator; do not re-import."
        ) from error
    try:
        opened = webbrowser.open(url, new=2)
    except (OSError, webbrowser.Error, KeyboardInterrupt):
        opened = False
    if opened:
        print(
            "Opened the Playground. No experiment has started; review the model and click Start in LangSmith."
        )
    else:
        print(
            f"Could not open a browser. Import is complete and the evaluator is enabled. Open this link manually: {url}"
        )
        print(
            "No experiment has started. Review the model and click Start in LangSmith."
        )


def import_package(folder, dataset_name=None, profile=None, model_config=None):
    folder = Path(folder).resolve()
    model_path = Path(model_config).resolve() if model_config else None
    config = json.loads((folder / "evaluator.json").read_text(encoding="utf-8-sig"))
    if (
        not isinstance(config, dict)
        or not isinstance(config.get("name"), str)
        or not config["name"].strip()
    ):
        raise ValueError("evaluator.json needs a nonempty name.")
    if dataset_name is None:
        dataset_name = (
            f"{config['name'].strip()} - {datetime.now(timezone.utc):%Y%m%d-%H%M%S-%f}"
        )
    judge = config.get("judge_type")
    required = [folder / "README.md", folder / "dataset.json"]
    if judge == "llm":
        required += [
            folder / name for name in ("prompt.json", "rubric.json", "variables.json")
        ]
        if model_path:
            required.append(model_path)
    elif isinstance(judge, str) and judge in {"deterministic", "code"}:
        filename, function = config.get("file"), config.get("function")
        if (
            filename not in ("evaluator.py", "evaluator.js", "evaluator.ts")
            or not isinstance(function, str)
            or not function.isidentifier()
        ):
            raise ValueError(
                "Declare a supported evaluator file and function in evaluator.json."
            )
        required.append(folder / filename)
        evaluator = [
            "evaluator",
            "upload",
            str(folder / filename),
            "--function",
            function,
        ]
    else:
        raise ValueError("judge_type must be deterministic, llm, or code.")
    native = {}
    for path in required:
        if not path.is_file() or not path.stat().st_size:
            raise ValueError(f"Required file is missing or empty: {path}")
        if path.suffix == ".json" or path == model_path:
            native[path] = json.loads(path.read_text(encoding="utf-8-sig"))

    task_path = folder / "task.json"
    task = None
    if task_path.exists() or task_path.is_symlink():
        if (
            task_path.is_symlink()
            or not task_path.is_file()
            or not 0 < task_path.stat().st_size <= BRAINTRUST_FILE_BYTES
        ):
            raise ValueError(
                "langchain/task.json must be a regular nonempty file up to 5 MiB."
            )
        try:
            task = langsmith_task(
                native_json(task_path.read_text(encoding="utf-8-sig")),
                native[folder / "dataset.json"],
            )
        except (ValueError, RecursionError) as error:
            raise ValueError(f"Invalid langchain/task.json: {error}") from None

    if judge == "llm":
        prompt = native[folder / "prompt.json"]
        if not isinstance(prompt, list) or not prompt:
            raise ValueError("prompt.json needs a nonempty array of messages.")
        messages = [
            [m.get("role"), m.get("content")] if isinstance(m, dict) else m
            for m in prompt
        ]
        if any(
            not isinstance(m, list)
            or len(m) != 2
            or not all(isinstance(v, str) and v.strip() for v in m)
            for m in messages
        ):
            raise ValueError(
                "prompt.json needs [role, content] pairs or message objects."
            )
        schema = native[folder / "rubric.json"]
        variables = native[folder / "variables.json"]
        if (
            not isinstance(schema, dict)
            or not schema
            or not isinstance(variables, dict)
        ):
            raise ValueError("rubric.json and variables.json need native JSON objects.")
        if any(not isinstance(value, str) for value in variables.values()):
            raise ValueError("variables.json must map names to field-path strings.")
        # Match the official CLI's native schema envelope; preserve seller fields.
        if "parameters" not in schema:
            schema = {"name": "eval", "description": "", "parameters": schema}
        elif not isinstance(schema["parameters"], dict):
            raise ValueError(
                "rubric.json parameters must contain a JSON schema object."
            )
        structured = {
            "prompt": messages,
            "schema": schema,
            "variable_mapping": variables,
            "template_format": "mustache",
        }
        if model_path:
            model = native[model_path]
            if not isinstance(model, dict) or not model:
                raise ValueError(
                    "The model configuration must be a nonempty JSON object."
                )
            structured["model"] = model

    executable = shutil.which("langsmith")
    if not executable:
        raise ValueError(
            "Install the official LangSmith CLI, then run this importer again: "
            "https://docs.langchain.com/langsmith/langsmith-cli"
        )
    command = [executable, "--format", "json"] + (
        ["--profile", profile] if profile else []
    )

    def run(arguments, body=None):
        # Keep files unchanged and delegate authentication/import behavior to the CLI.
        try:
            result = subprocess.run(
                command + arguments,
                check=True,
                stdout=subprocess.PIPE,
                text=True,
                encoding="utf-8",
                input=json.dumps(body) if body is not None else None,
            )
        except subprocess.CalledProcessError as error:
            # Raw API errors are printed on stdout by the CLI. Preserve them
            # so the user can see why an import failed, not just its exit code.
            if error.stdout:
                print(error.stdout, file=sys.stderr)
            raise
        return json.loads(result.stdout)

    auth = run(["auth", "info"])
    if not isinstance(auth, dict) or not isinstance(auth.get("authenticated"), bool):
        raise ValueError("Could not read the LangSmith CLI's authentication state.")
    if not auth["authenticated"] or (
        auth.get("oauth_expired") is True and not auth.get("oauth_refresh_token")
    ):
        print(
            "Sign in to LangSmith in your browser to continue the import.", flush=True
        )
        login = [executable, "--format", "pretty"] + (
            ["--profile", profile] if profile else []
        )
        # Inherit the terminal so the official CLI can show its login link and
        # handle workspace selection. It stores and manages its own login.
        subprocess.run(login + ["auth", "login", "--prompt-workspace"], check=True)
        auth = run(["auth", "info"])
        if not isinstance(auth, dict) or auth.get("authenticated") is not True:
            raise ValueError(
                "LangSmith sign-in did not complete. Nothing has been uploaded."
            )

    if judge == "llm" and not model_path:
        # Reuse the saved native configuration unchanged. The API requires both
        # its model settings and ID; no model file or provider conversion is needed.
        structured.update(choose_model(run))

    print(
        f"Uploading dataset {dataset_name!r} using your current LangSmith account...",
        flush=True,
    )
    dataset = run(
        ["dataset", "upload", str(folder / "dataset.json"), "--name", dataset_name]
    )
    dataset_id = dataset.get("dataset_id") if isinstance(dataset, dict) else None
    if not isinstance(dataset_id, str) or not dataset_id:
        raise ValueError(
            "The CLI did not return a dataset ID. Inspect LangSmith before retrying."
        )
    print(f"Created dataset: {dataset_id}", flush=True)
    # Zero sampling installs the evaluator without automatically grading new runs.
    if judge == "llm":
        # The CLI's create-llm command requires a file; its authenticated API
        # command also supports LangSmith's native saved-model reference.
        evaluator_result = run(
            ["api", "runs/rules", "--method", "POST", "--input", "-"],
            {
                "display_name": config["name"],
                "dataset_id": dataset_id,
                "sampling_rate": 0,
                "is_enabled": True,
                "include_extended_stats": False,
                "evaluators": [{"structured": structured}],
            },
        )
    else:
        evaluator_result = run(
            evaluator
            + [
                "--name",
                config["name"],
                "--dataset",
                dataset_id,
                "--sampling-rate",
                "0",
            ]
        )
    print("Imported dataset and evaluator. No evaluation was run. Sampling is 0.")
    if task is not None:
        dataset_id = langsmith_id(dataset_id, "dataset")
        field = "id" if judge == "llm" else "rule_id"
        rule_id = langsmith_id(
            evaluator_result.get(field) if isinstance(evaluator_result, dict) else None,
            "evaluator rule",
        )
        print(f"Created evaluator rule: {rule_id}", flush=True)
        url = import_langsmith_task(run, task, dataset_id, auth)
        print("Import complete. No experiment has started.")
        if url:
            offer_langsmith_playground(run, url, dataset_id, rule_id)
    else:
        print(
            "This package has no task.json for a prepared Playground. Ask the seller for the Playground task file, or follow the package's included task setup and enable grading in LangSmith when ready."
        )
    return dataset_id


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "folder",
        help="Extracted platform folder: braintrust or langchain (or a path to it)",
    )
    parser.add_argument("--project-id", help="Braintrust destination project UUID")
    parser.add_argument(
        "--yes", action="store_true", help="Confirm the displayed Braintrust import"
    )
    parser.add_argument(
        "--provider",
        action="append",
        help="Braintrust buyer provider selection: function-slug=discovered-provider-name (required when the model is ambiguous)",
    )
    parser.add_argument(
        "--model",
        action="append",
        help="Braintrust buyer model selection: function-slug=discovered-model-name (required per prompt task/LLM scorer in unattended imports)",
    )
    parser.add_argument(
        "--dataset",
        help="Optional dataset name; defaults to the evaluator name plus a unique timestamp",
    )
    parser.add_argument(
        "--profile",
        help="Existing LangSmith CLI profile; otherwise use the CLI's current authentication",
    )
    parser.add_argument(
        "--model-config",
        help="Optional existing native model JSON; otherwise choose a saved LangSmith model interactively",
    )
    args = parser.parse_args()
    try:
        if Path(args.folder).name == "braintrust":
            if args.dataset or args.profile or args.model_config:
                raise ValueError(
                    "--dataset, --profile and --model-config are LangSmith-only options."
                )
            import_braintrust(
                args.folder, args.project_id, args.yes, args.model, args.provider
            )
        else:
            if args.project_id or args.yes or args.model or args.provider:
                raise ValueError("Braintrust options require a braintrust/ folder.")
            import_package(args.folder, args.dataset, args.profile, args.model_config)
    except PlaygroundSetupError as error:
        print(
            f"\nImport complete; Playground setup needs attention.\n{error}",
            file=sys.stderr,
        )
        return 1
    except ExperimentStartError as error:
        print(
            f"\nImport complete; experiment start needs attention.\n{error}",
            file=sys.stderr,
        )
        return 1
    except (
        OSError,
        ValueError,
        subprocess.CalledProcessError,
        KeyboardInterrupt,
        EOFError,
    ) as error:
        print(
            f"Import stopped: {error}\nIf an upload started, inspect your destination workspace for partial resources before retrying; this helper does not retry or replace existing resources.",
            file=sys.stderr,
        )
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
