From 1724cb26379a4fe87b70f0fb06e2ac61c40aeb55 Mon Sep 17 00:00:00 2001 From: Ole-Morten Duesund Date: Tue, 25 Aug 2026 15:14:54 +0200 Subject: [PATCH] conandeps: always show the binary's source; cache-only binaries are not available The tool's question is "does the remote have this binary?", so local cache state must not mask the answer. Conan reports Cache for a locally present binary without consulting the remote; for those nodes we now run a package search on the remote(s) and only count the binary as available if the same package_id is found there. Binaries that exist only in the local cache are rendered as "not available", listed under missing binaries with a note, and make the exit status 1. Every available cell now names the remote that has the binary instead of a bare "ok". Adds a test that builds a package locally without uploading it and checks it is reported as cache-only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EYNdaGDrpaqDyT8QtrTQbM --- CLAUDE.md | 6 +++ README.md | 31 ++++++++----- conandeps.py | 98 ++++++++++++++++++++++++++++++++--------- tests/test_conandeps.py | 33 +++++++++++++- 4 files changed, 135 insertions(+), 33 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d0ee767..7024f69 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,12 @@ for what it does and how it is used. replace this with settings-dict matching against `conan search` output. - **"Missing" means missing for the exact profile.** Not "no binary with that build_type at all". +- **Only the remote counts, and the source is always shown.** Conan says + `Cache` for a locally cached binary without asking the remote, so + `_RemoteIndex` verifies cache hits with `search_packages` on the remote. + Cache-only binaries are reported as `not available` and count as a + problem (exit 1). Every available cell names the remote. The user rejected + an opt-in flag for this – it is the tool's core question. - **Overrides are parsed from Conan's WARN output.** Conan 1 does not record overrides on the graph; `Requirements.update()` mutates `req.ref` in place and only emits `": requirement overridden by to "`. diff --git a/README.md b/README.md index de5e9ea..4a7f284 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,11 @@ Given a `conanfile.py`, `conandeps` tells you three things in one report: 1. **What you depend on** – direct and indirect requirements, host and build context, as a table and as a tree. 2. **Which binaries are missing** – for every dependency, whether a package - exists on the remote for **Release, Debug and RelWithDebInfo** (or any list - you choose), evaluated for *your exact profile* (compiler, version, libcxx, - arch, options, …), not "some binary with that build type". + exists *on the remote* for **Release, Debug and RelWithDebInfo** (or any + list you choose), evaluated for *your exact profile* (compiler, version, + libcxx, arch, options, …), not "some binary with that build type". Every + cell names the remote that has the binary; a binary that only lives in + your local cache is reported as not available. 3. **Which requirements were overridden** – both explicit `self.requires("x/2.0", override=True)` and implicit ones, where a downstream consumer simply asks for a newer version than a transitive @@ -91,10 +93,10 @@ Dependencies (4) and binary availability package kind ctx Release Debug RelWithDebInfo ------------------------------------------------------------------------- -MyDepA/1.0 direct host ok ok ok -boost/1.8 indir. host ok ok ok -MyDepB/1.0 direct host ok ok ok -Zigma/1.0 indir. host ok ok MISSING +MyDepA/1.0 direct host knor knor knor +boost/1.8 indir. host knor knor knor +MyDepB/1.0 direct host knor knor knor +Zigma/1.0 indir. host knor knor MISSING MISSING BINARIES (1 packages) Zigma/1.0: missing RelWithDebInfo @@ -115,10 +117,12 @@ MyProject/conanfile.py Reading the table: -* `ok` – a binary for your profile exists (in the remote, or already in your - cache). -* `MISSING` – no binary for the package_id your profile produces. Hover the - cell in the HTML report to see the package_id. +* `knor` (a remote name) – the remote has a binary for the package_id your + profile produces. Hover a cell in the HTML report to see the package_id. +* `not available` – the binary exists only in your local Conan cache; the + remote does not have it, so a clean machine or CI would fail. Counts as + missing. +* `MISSING` – no binary anywhere. * `-` – not applicable (Conan marked the node `Skip` or `Editable`). * Header-only packages show `ok` everywhere: their package_id ignores `build_type`. @@ -142,7 +146,10 @@ dependency graph is built once per build type through Conan's own `info` API – the same code path `conan install` uses – and each node's binary status (`Cache`, `Download`, `Update`, `Missing`, …) is read back. That is why `package_id()` customisations, options and `default_package_id_mode` are -honoured exactly as a real install would. +honoured exactly as a real install would. Conan reports `Cache` without +consulting the remote, so for those nodes the tool additionally runs a +package search on the remote and only counts the binary as available if the +same package_id is found there. Conan 1.x does not keep override information on the graph object; the only trace is a `WARN: : requirement A overridden by B to C` line written while diff --git a/conandeps.py b/conandeps.py index 71b3977..8fddd95 100755 --- a/conandeps.py +++ b/conandeps.py @@ -61,8 +61,14 @@ _OVERRIDE_RE = re.compile( r"requirement (?P\S+) overridden by (?Pyour conanfile|\S+) to (?P\S+)" ) -# Binary states that mean "a usable binary exists" (see conans/client/graph/graph.py). -_AVAILABLE = {"Cache", "Download", "Update"} +# Binary states, after our post-processing of Conan's node.binary +# (see conans/client/graph/graph.py for Conan's own set): +# Remote - the remote has the binary (Conan said Download/Update, or said +# Cache and we confirmed the remote has the same package_id) +# CacheOnly - only the local cache has it; a clean machine would fail +# Missing - nowhere +_AVAILABLE = {"Remote"} +_PROBLEM = {"Missing", "CacheOnly"} # States that mean "does not apply", not "missing". _NOT_APPLICABLE = {"Skip", "Editable"} @@ -85,7 +91,8 @@ class Dep: return self.ref.split("/", 1)[0] def missing(self) -> List[str]: - return [bt for bt, (status, _, _) in self.binaries.items() if status == "Missing"] + """Build types without a binary on the remote (cache-only counts as missing).""" + return [bt for bt, (status, _, _) in self.binaries.items() if status in _PROBLEM] @dataclass @@ -199,7 +206,47 @@ def _parse_overrides(text: str, explicit: set) -> List[Override]: return result -def _collect(graph, build_type: str, deps: "OrderedDict[str, Dep]", ranges: Dict[str, str]): +class _RemoteIndex: + """Answers "does remote X have package_id Y of ref Z?" with one search per ref. + + Conan reports ``Cache`` for a binary it already has locally without ever + asking the remote. The question this tool answers is whether the *remote* + has it, so cache hits are verified here via ``conan search`` on the remote + (or all remotes when none was given). + """ + + def __init__(self, api, remote: Optional[str]): + self._api = api + self._remote = remote or "all" + self._cache: Dict[str, Dict[str, str]] = {} # ref -> {package_id: remote_name} + + def remote_for(self, ref: str, package_id: str) -> Optional[str]: + if ref not in self._cache: + self._cache[ref] = self._search(ref) + return self._cache[ref].get(package_id) + + def _search(self, ref: str) -> Dict[str, str]: + from conans.errors import ConanException + + found: Dict[str, str] = {} + try: + info = self._api.search_packages(ref, remote_name=self._remote) + except ConanException: + return found # recipe not on the remote at all + for remote_info in info.get("results", []): + for item in remote_info.get("items", []): + for pkg in item.get("packages", []): + found.setdefault(pkg["id"], remote_info["remote"]) + return found + + +def _collect( + graph, + build_type: str, + deps: "OrderedDict[str, Dep]", + ranges: Dict[str, str], + index: _RemoteIndex, +): """Merge one build_type's graph into ``deps``. The graph may differ between build types (conditional requirements), so we @@ -231,10 +278,16 @@ def _collect(graph, build_type: str, deps: "OrderedDict[str, Dep]", ranges: Dict except Exception: # header-only / no settings dep.settings = {} status = node.binary or "Unknown" + package_id = node.package_id or "" + remote = node.binary_remote.name if node.binary_remote else None if status in _NOT_APPLICABLE: status = "n/a" - remote = node.binary_remote.name if node.binary_remote else None - dep.binaries[build_type] = (status, node.package_id or "", remote) + elif status in ("Download", "Update"): + status = "Remote" + elif status == "Cache": + remote = index.remote_for(str(node.ref), package_id) + status = "Remote" if remote else "CacheOnly" + dep.binaries[build_type] = (status, package_id, remote) def build_report( @@ -257,6 +310,7 @@ def build_report( root_requires: List[str] = [] profile_settings: Dict[str, str] = {} explicit: set = set() + index = _RemoteIndex(api, remote) total = len(build_types) for i, bt in enumerate(build_types, 1): @@ -275,8 +329,8 @@ def build_report( update=update, ) explicit |= _explicit_overrides(graph) - _collect(graph, bt, deps, ranges) - missing = sum(1 for d in deps.values() if d.binaries.get(bt, ("",))[0] == "Missing") + _collect(graph, bt, deps, ranges, index) + missing = sum(1 for d in deps.values() if d.binaries.get(bt, ("",))[0] in _PROBLEM) _progress( quiet, "[%d/%d] %s: %d packages, %d missing binaries (%.1fs)" @@ -312,9 +366,7 @@ def build_report( # Text rendering # --------------------------------------------------------------------------- # _MARK = { - "Cache": "ok", - "Download": "ok", - "Update": "ok", + "CacheOnly": "not available", "Missing": "MISSING", "Build": "build", "n/a": "-", @@ -369,6 +421,13 @@ def _tree_lines(report: Report) -> List[str]: return lines +def _cell(status: str, _package_id: str, remote: Optional[str]) -> str: + """Cell text: the remote that has the binary, or why it is not available.""" + if status == "Remote": + return remote or "remote" + return _MARK.get(status, status) + + def render_text(report: Report) -> str: out: List[str] = [] w = out.append @@ -388,10 +447,7 @@ def render_text(report: Report) -> str: w("-" * len(header)) for dep in report.deps.values(): kind = "direct" if dep.direct else "indir." - cells = " ".join( - "%-14s" % _MARK.get(dep.binaries[bt][0], dep.binaries[bt][0]) - for bt in report.build_types - ) + cells = " ".join("%-14s" % _cell(*dep.binaries[bt]) for bt in report.build_types) w("%-*s %-6s %-4s %s" % (refw, dep.ref, kind, dep.context, cells)) w("") @@ -402,9 +458,11 @@ def render_text(report: Report) -> str: for dep in problems: w(" %s: missing %s" % (dep.ref, ", ".join(dep.missing()))) for bt in dep.missing(): - w(" %s package_id %s" % (bt, dep.binaries[bt][1])) + status, pid, _ = dep.binaries[bt] + note = " (in local cache only, not on the remote)" if status == "CacheOnly" else "" + w(" %s package_id %s%s" % (bt, pid, note)) else: - w("All packages have binaries for: " + ", ".join(report.build_types)) + w("All packages have binaries on the remote for: " + ", ".join(report.build_types)) w("") # ---- overrides ------------------------------------------------------- @@ -470,9 +528,9 @@ def render_html(report: Report) -> str: ) for bt in report.build_types: status, pid, remote = dep.binaries[bt] - cls = "ok" if status in _AVAILABLE else "missing" if status == "Missing" else "na" - title = ' title="package_id %s%s"' % (e(pid), " @ " + e(remote) if remote else "") - w('%s' % (cls, title, e(status))) + cls = "ok" if status in _AVAILABLE else "missing" if status in _PROBLEM else "na" + title = ' title="package_id %s"' % e(pid) + w('%s' % (cls, title, e(_cell(status, pid, remote)))) w("") w("") diff --git a/tests/test_conandeps.py b/tests/test_conandeps.py index 2333e08..4107066 100644 --- a/tests/test_conandeps.py +++ b/tests/test_conandeps.py @@ -228,11 +228,40 @@ def test_missing_binaries(report): for dep in report.deps.values(): for bt, (status, pid, remote) in dep.binaries.items(): if dep.ref != "libbar/1.1": - assert status == "Download", (dep.ref, bt, status) + assert status == "Remote", (dep.ref, bt, status) assert remote == "test" assert pid +def test_cache_only_is_not_available(conan_env): + """A binary that exists only in the local cache must not count as available. + + Build libbar/1.1 Debug locally without uploading it, and install the + Release one so it is cached too. Release must still be attributed to the + remote; Debug must be reported as cache-only (a problem). + """ + env = dict(os.environ) + d = Path(conan_env["home"]) / "libbar-local" + d.mkdir() + (d / "conanfile.py").write_text(RECIPE.format(name="libbar", version="1.1", requires="")) + _conan(env, "create", str(d), "-s", "build_type=Debug") + _conan(env, "install", "libbar/1.1@", "-s", "build_type=Release", "-r", "test") + + report = conandeps.build_report( + conan_env["conanfile"], "test", [], [], [], list(BUILD_TYPES), update=False, quiet=True + ) + libbar = report.deps["libbar#host"] + assert libbar.binaries["Release"][0] == "Remote" + assert libbar.binaries["Release"][2] == "test" + assert libbar.binaries["Debug"][0] == "CacheOnly" + assert libbar.binaries["RelWithDebInfo"][0] == "Missing" + assert libbar.missing() == ["Debug", "RelWithDebInfo"] + + text = conandeps.render_text(report) + assert "not available" in text and "in local cache only" in text + _conan(env, "remove", "libbar/1.1@", "-f") # leave the cache as other tests expect + + def test_overrides(report): ov = {(o.package, o.old, o.new, o.by, o.explicit) for o in report.overrides} assert ov == { @@ -244,6 +273,8 @@ def test_overrides(report): def test_text_and_html_render(report, tmp_path): text = conandeps.render_text(report) assert "libbar/1.1" in text and "MISSING" in text + table = text.split("MISSING BINARIES")[0] + assert table.count("test") >= 4 * 3 - 2 # every available cell names the remote assert "overrides libbar/1.0 (explicit" in text and "explicit override=True" in text assert "`-- " in text or "|-- " in text # tree drawn page = conandeps.render_html(report)