mirror of
https://github.com/Comfy-Org/ComfyUI.git
synced 2026-09-29 09:28:35 -05:00
* review-stack 1/4: code (37 files, +3217/-3958) Review-and-land stack for synap5e/feat/asset-record-content-split, generated by review-stack.py. Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: path not under tests-unit/ or tests/ Question: Is the logic change right? Source tip:7007d18582Merge-base:783545f689* review-stack 2/4: tests-removed (24 files, +274/-8220) Review-and-land stack for synap5e/feat/asset-record-content-split, generated by review-stack.py. Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: test file deleted, or modified with deleted/(added+deleted) >= 0.9 Question: For each dropped assertion: obsolete by a ruling, or covered by a tests-new test? Source tip:7007d18582Merge-base:783545f689* review-stack 3/4: tests-changed (13 files, +1043/-1218) Review-and-land stack for synap5e/feat/asset-record-content-split, generated by review-stack.py. Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: remaining modified test files (incl. conftest.py / helpers) Question: Did the edits weaken an existing check? Source tip:7007d18582Merge-base:783545f689* review-stack 4/4: tests-new (46 files, +8601/-0) Review-and-land stack for synap5e/feat/asset-record-content-split, generated by review-stack.py. Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: test file added Question: Is the code layer well covered? Source tip:7007d18582Merge-base:783545f689* review-stack 5/6: code (13 files, +351/-104) Review-and-land stack for synap5e/feat/assets-di, generated by review-stack.py. Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: path not under tests-unit/ or tests/ Question: Is the logic change right? Source tip:eca2c74bffMerge-base:20d59d2a5f* review-stack 6/6: tests (8 files, +753/-238) Review-and-land stack for synap5e/feat/assets-di, generated by review-stack.py. Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: every changed file under tests-unit/ or tests/ (added, modified, or deleted) Question: Is the code layer well covered, and did any edit weaken an existing check? Source tip:eca2c74bffMerge-base:20d59d2a5f* review-stack 7/8: ported-fixes (42 files, +1361/-180) Review-and-land stack for synap5e/feat/assets-di-v2, generated by review-stack.py conventions (hand-built continuation layer; see the PR body). Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: the 11 base-branch fix/docs commits 595cd6e4..94d7185b cherry-picked across the DI refactor (7efdd1d7excluded, superseded by layer 8) Question: was each base fix ported faithfully across the DI refactor? Source tip: 6841881069284803b902b4a9e33bdcda13126771 Merge-base:7fdfb40f4b* review-stack 8/8: defensive-parity (4 files, +36/-3) Review-and-land stack for synap5e/feat/assets-di-v2, generated by review-stack.py conventions (hand-built continuation layer; see the PR body). Once approved, merges DOWN into the layer below (a fast-forward); only the bottom layer squash-merges into the real base. See ~/adocs/review-stack.md. Rule: match-or-improve master's dependency defenses — NoAssets selection when DB deps unavailable (7efdd1d7's outcome via the DI seam), requirements warning before assets imports, blake3 in the guarded dependency set Question: does each degradation path now match or improve master's behavior? Source tip:ebc2cfeebcMerge-base:7fdfb40f4b* fix(assets): only discard content rows this operation actually inserted CR-9: Enumerated all six create_content call sites. Only scanner seeding and the three ingest registration paths track IDs for failure cleanup. * fix(assets): reject hash-only uploads with FEATURE_DISABLED when hashing is off CodeRabbit finding CR-2: reject hash-only multipart uploads before create_from_hash when hashing is disabled. * fix(assets): seed persists the stat it verified CR-7: persist the fresh seed-time restat instead of walk-time spec values. * fix(assets): route database lock failures to the lock guidance CR-16: route file-lock startup failures through the existing lock guidance and exit path. * fix(assets): drop the inaccurate temp-cleanup claim from the shutdown warning References CR-10. * fix(assets): walk the output root after execution so undeclared outputs register promptly Custom nodes that write files into the output directory without declaring them in output_ui only became assets when the next full walk happened - a frontend GET /object_info or a restart. Headless and API-only sessions never trigger either, so those files never converged into the asset database. The post-execution hook now requests a FULL scan of the output root instead of an enrich-only pass. The seeder's pending-request queue was generalised from enrich-specific to carrying a scan phase, so the request starts immediately when the seeder is idle and coalesces (escalating to FULL on a phase mismatch) when a scan is already running. queue_output_enrichment is renamed to queue_output_scan across the protocol, the NoAssets no-op and the call site. References FIX-6. * chore(assets): remove seeder paths orphaned by the output-scan change 45c2f96e rerouted both former enrich call sites to start()/enqueue_scan(), leaving two seeder methods that look live but are not. Review round F2 raised this along with four smaller items; the user's disposition was to fix all six here. - Delete start_enrich: zero callers repo-wide after 45c2f96e. - Delete enqueue_enrich: no production callers; its ~18 call sites in tests/test_asset_seeder.py move to enqueue_scan(phase=ScanPhase.ENRICH) with their semantics unchanged. The deletion forces the half-done class renames (TestEnqueueEnrich* -> TestEnqueueScan*, consistent with the already-renamed TestPendingScanDrain) and restores the module docstring that was dropped rather than reworded. - Document at manager.queue_output_scan that ScanPhase.FULL per debounce window is the deliberate, user-ratified trade, so it is not optimised back to ENRICH without revisiting the decision. - Document that SeedAssetSpec.size_bytes/mtime_ns are walk-time diagnostics only - production persists the seed-time restat since CR-7. - Export create_content_reporting_insert from the queries facade and fold scanner.py's direct-module import into the existing facade block. - Harden test_queue_output_scan_does_not_duplicate_declared_output against a vacuous pass: it now asserts the seeder finished without errors and that an undeclared sibling written into the same directory WAS registered by the same scan, proving the walk actually ran. No production behaviour changes beyond the two deletions. References F2-cleanup. * chore: comment cleanup Comment-Gate: 18 quarantined * fix(assets): preserve pause across the seeder's pending-scan drain pause() runs before every prompt, while pending-scan enqueue and resume only run inside the debounced gc-interval gate. If the active scan finishes just after the next prompt's pause, its finally block resets the seeder to idle and the pending drain starts a replacement with the run gate open, so resume becomes a no-op. Capture pausedness under the lock before resetting to idle, then start the drained scan already paused. Setting the state and gate before launching the thread avoids the start-then-reclear window and lets resume release the existing scan checkpoints. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> * test(assets): pin job_id absence for scan-discovered assets Owner ruling, recorded 2026-09-03 in the stack-9-hardening planning notepad: scan-discovered assets — including undeclared outputs found by the post-execution walk — carry job_id = None, always; only emission-time registration (output_ui declaration) attributes a job; attributing walk finds to the most recent prompt would be a temporal-correlation guess that is wrong exactly when prompts interleave; None is honest provenance. Do NOT add proximity-based attribution heuristics to the scanner. Ratified against Jacob Segal's cross-job-attribution concern (2026-09-08 review meeting) — a wrongly-attributed asset could mean one user's cloud job sees another user's asset. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> * [review-stack 10/10] assets-tests (#16218) * test(execution): run the battery with assets enabled and assert asset-system health at teardown * test(execution): cover list-shaped outputs registering assets Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> --------- Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> * [review-stack 11/11] review-fixes (#16261) * fix(assets): only exit on database file-lock timeout when assets are enabled * test(assets): pin live_contents_under_prefixes path-filtering semantics * perf(assets): push live-content prefix filtering into SQL * test(assets): declare per-entry intent in the path-prefix corpus * test(assets): normalize POSIX-literal path expectations for Windows * test(assets): force observable stat changes and close-before-mutate on Windows-sensitive rewrites * test(assets): force an observable mtime change in the hash-mode split test --------- Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai> Co-authored-by: guill <jacob.e.segal@gmail.com>
153 lines
4.5 KiB
Python
153 lines
4.5 KiB
Python
"""Runs the asset system's startup and shutdown maintenance: clearing temp rows
|
|
and files, recording a hash-mode transition, and handing the filesystem scan to
|
|
the background seeder. Startup only enqueues transition work — draining it
|
|
belongs to the seeder, so a large library cannot stall the server before it
|
|
accepts requests. It also settles, before any of that, whether the database
|
|
dependencies exist at all, disabling the asset routes when they do not.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
|
|
import folder_paths
|
|
from sqlalchemy import select
|
|
|
|
from app.assets.database.models import Asset, AssetContent
|
|
from app.assets.database.queries.records import delete_record
|
|
from app.assets.helpers import sql_path_under_prefix
|
|
from app.assets.services.hash_mode_state import enqueue_transition_work
|
|
from app.assets.services.hash_mode_state import record_transition_intent
|
|
from app.database.db import can_create_session, create_session
|
|
from comfy.cli_args import args
|
|
|
|
_excluded_scan_roots: set[str] = set()
|
|
_hash_mode_transition: str | None = None
|
|
|
|
|
|
def get_excluded_scan_roots() -> frozenset[str]:
|
|
return frozenset(_excluded_scan_roots)
|
|
|
|
|
|
def record_hash_mode_transition_intent() -> None:
|
|
global _hash_mode_transition
|
|
|
|
with create_session() as session:
|
|
_hash_mode_transition = record_transition_intent(session)
|
|
session.commit()
|
|
|
|
|
|
def enqueue_mode_transition_work() -> None:
|
|
with create_session() as session:
|
|
enqueue_transition_work(session, _hash_mode_transition)
|
|
session.commit()
|
|
|
|
|
|
def wipe_temp_db_rows(session) -> tuple[int, int]:
|
|
try:
|
|
temp_root = os.path.abspath(folder_paths.get_temp_directory())
|
|
except OSError:
|
|
return 0, 0
|
|
# These rows are hard-deleted, so the predicate must stay case-SENSITIVE: admitting a
|
|
# case-different persistent directory destroys user assets.
|
|
under_temp = sql_path_under_prefix(AssetContent.path, temp_root)
|
|
|
|
temp_record_ids = list(
|
|
session.scalars(
|
|
select(Asset.id)
|
|
.join(AssetContent, Asset.content_id == AssetContent.id)
|
|
.where(under_temp)
|
|
)
|
|
)
|
|
|
|
records_deleted = 0
|
|
for record_id in temp_record_ids:
|
|
delete_record(session, record_id)
|
|
records_deleted += 1
|
|
|
|
contents_deleted = 0
|
|
for content in session.scalars(select(AssetContent).where(under_temp)).all():
|
|
session.delete(content)
|
|
contents_deleted += 1
|
|
|
|
session.flush()
|
|
return records_deleted, contents_deleted
|
|
|
|
|
|
def cleanup_temp_filesystem() -> bool:
|
|
temp_dir = os.path.abspath(folder_paths.get_temp_directory())
|
|
if not os.path.exists(temp_dir):
|
|
return True
|
|
try:
|
|
shutil.rmtree(temp_dir)
|
|
return True
|
|
except OSError as exc:
|
|
logging.warning(
|
|
"Failed to remove temp directory %s: %s — excluding from scan for this process",
|
|
temp_dir,
|
|
exc,
|
|
)
|
|
_excluded_scan_roots.add(temp_dir)
|
|
return False
|
|
|
|
|
|
def start_asset_seeder() -> bool:
|
|
from app.assets.seeder import asset_seeder
|
|
|
|
started = asset_seeder.start(
|
|
roots=("models", "input", "output"),
|
|
prune_first=True,
|
|
compute_hashes=args.enable_asset_hashing,
|
|
)
|
|
if started:
|
|
logging.info("Background asset scan initiated for models, input, output")
|
|
return started
|
|
|
|
|
|
def run_asset_startup() -> None:
|
|
try:
|
|
with create_session() as session:
|
|
wipe_temp_db_rows(session)
|
|
session.commit()
|
|
except Exception:
|
|
logging.exception("Temp DB row wipe failed; skipping filesystem cleanup")
|
|
enqueue_mode_transition_work()
|
|
start_asset_seeder()
|
|
return
|
|
cleanup_temp_filesystem()
|
|
enqueue_mode_transition_work()
|
|
start_asset_seeder()
|
|
|
|
|
|
def run_startup(*, enable_assets: bool) -> None:
|
|
try:
|
|
if enable_assets:
|
|
run_asset_startup()
|
|
else:
|
|
cleanup_temp_filesystem()
|
|
except Exception:
|
|
logging.exception("Asset startup maintenance failed")
|
|
|
|
|
|
def run_asset_shutdown_cleanup() -> None:
|
|
try:
|
|
with create_session() as session:
|
|
wipe_temp_db_rows(session)
|
|
session.commit()
|
|
except Exception:
|
|
logging.exception("Temp DB row wipe failed during shutdown")
|
|
finally:
|
|
cleanup_temp_filesystem()
|
|
|
|
|
|
def run_shutdown() -> None:
|
|
try:
|
|
if can_create_session():
|
|
run_asset_shutdown_cleanup()
|
|
else:
|
|
cleanup_temp_filesystem()
|
|
except Exception:
|
|
logging.exception("Asset shutdown cleanup failed")
|