Sweep old items from the temp folder at the start of each job
This commit is contained in:
1 parent
be36cf323f
commit
faea82a0b1
1 file changed
+21
@@ -454,12 +454,33 @@ def apply_paths(s):
|
|||||||
return models, temp
|
return models, temp
|
||||||
|
|
||||||
|
|
||||||
|
def sweep_temp(temp, max_age_hours=1):
|
||||||
|
cutoff = datetime.now().timestamp() - max_age_hours * 3600
|
||||||
|
removed = 0
|
||||||
|
for item in temp.iterdir():
|
||||||
|
if item.name.startswith("torchinductor_") or item.name.startswith(".write_test"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if item.lstat().st_mtime > cutoff:
|
||||||
|
continue
|
||||||
|
if item.is_dir() and not item.is_symlink():
|
||||||
|
shutil.rmtree(item, ignore_errors=True)
|
||||||
|
else:
|
||||||
|
item.unlink(missing_ok=True)
|
||||||
|
removed += 1
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if removed:
|
||||||
|
logger.info("Cleaned %d old items from the temp folder", removed)
|
||||||
|
|
||||||
|
|
||||||
def prepare_storage(s):
|
def prepare_storage(s):
|
||||||
root = validate_storage(s["storage_dir"])
|
root = validate_storage(s["storage_dir"])
|
||||||
free = free_gb(root)
|
free = free_gb(root)
|
||||||
if free is not None and free < MIN_FREE_GB:
|
if free is not None and free < MIN_FREE_GB:
|
||||||
raise RuntimeError(f"Only {free:.1f} GB free at {root}, need at least {MIN_FREE_GB} GB")
|
raise RuntimeError(f"Only {free:.1f} GB free at {root}, need at least {MIN_FREE_GB} GB")
|
||||||
models, temp = apply_paths(s)
|
models, temp = apply_paths(s)
|
||||||
|
sweep_temp(temp)
|
||||||
logger.info("Storage: %s (%.1f GB free) | Models: %s | Temp: %s", root, free or 0, models, temp)
|
logger.info("Storage: %s (%.1f GB free) | Models: %s | Temp: %s", root, free or 0, models, temp)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user