diff --git a/apps/game/tools/aiaudio/__pycache__/post.cpython-311.pyc b/apps/game/tools/aiaudio/__pycache__/post.cpython-311.pyc new file mode 100644 index 0000000..095305a --- /dev/null +++ b/apps/game/tools/aiaudio/__pycache__/post.cpython-311.pyc Binary files differ diff --git a/apps/game/tools/aiaudio/gen_audioldm2.py b/apps/game/tools/aiaudio/gen_audioldm2.py index 37e131e..485b679 100644 --- a/apps/game/tools/aiaudio/gen_audioldm2.py +++ b/apps/game/tools/aiaudio/gen_audioldm2.py @@ -1,14 +1,16 @@ -"""PoC (работа A, этап A0): text-to-audio через AudioLDM2 на CPU. +"""PoC (работа A, этап A0): text-to-audio через AudioLDM2. Главный кандидат плана — Stable Audio Open Small — gated (нужен HF-токен), поэтому PoC делаем на фолбэке AudioLDM2 (cvssp/audioldm2, CC-BY-NC-SA — в ОСНОВНОЙ пайплайн с некоммерческими весами не пойдёт, см. docs/plan.md). Запуск: - ~/.cache/rpg-ai/venv-pxgpt/bin/python tools/aiaudio/gen_audioldm2.py [seed] + gen_audioldm2.py # все клипы из LIST с seed по умолчанию + gen_audioldm2.py [id[:seed]...] # один/несколько клипов +Клип = запись LIST (промпт, длительность, шаги); выход — review/_.wav. -Выход: tools/aiaudio/review/_.wav (16000 Гц — ресэмпл делает post). -Детерминизм: seed + num_inferences фиксированы, промпт в манифесте. +Детерминизм: seed + шаги фиксированы, промпт в манифесте; детерминизм +по-устройству (fp16 GPU ≠ fp32 CPU). GPU (RTX 3090) при наличии: fp16. """ import sys from pathlib import Path @@ -19,19 +21,23 @@ HERE = Path(__file__).resolve().parent REVIEW = HERE / "review" -# Промпты PoC: амбиент прудов (туман, вода) против текущего процедурного -# gen.mjs (band-noise ветер + капли, 8 с) — критерий «лучше на слух». -PROMPTS = { +NEG_NOISE = ( + "music, melody, rhythm, drums, speech, voices, sudden loud noises, " + "harsh noise, distortion" +) + +# Клипы PoC: амбиенты (длинные, лупуются пост-обработкой) и SFX (короткие, +# без лупа). Эталоны процедурных — tools/audio/gen.mjs и data/sfxSpecs.ts. +LIST = { + # --- амбиенты (10 с, луп) --- "amb_ponds": { "positive": ( "field recording, calm misty pond at dusk, gentle water lapping, " "soft wind through reeds, occasional distant croaking, eerie " "quiet melancholic atmosphere, seamless ambience" ), - "negative": ( - "music, melody, rhythm, drums, speech, voices, sudden loud " - "noises, harsh noise, distortion" - ), + "negative": NEG_NOISE, + "duration": 10.0, }, "amb_zvenets": { "positive": ( @@ -39,66 +45,150 @@ "metallic bell toll, soft wind, faint rustling, somber " "melancholic atmosphere, seamless ambience" ), - "negative": ( - "music, melody, song, rhythm, drums, speech, voices, sirens, " - "harsh noise, distortion" + "negative": NEG_NOISE, + "duration": 10.0, + }, + "amb_meadows": { + "positive": ( + "field recording, desolate burned field, cold dry wind blowing " + "over dead grass, occasional faint debris rattling, bleak " + "post-fire emptiness, seamless ambience" ), + "negative": NEG_NOISE, + "duration": 10.0, + }, + "amb_ponds_night": { + "positive": ( + "field recording, pond at night, deeper water lapping, night " + "insects, distant eerie echoes, dark oppressive quiet, " + "seamless ambience" + ), + "negative": NEG_NOISE, + "duration": 10.0, + }, + "amb_tower_hum": { + "positive": ( + "deep metallic resonance hum, giant bell tower vibrating, low " + "sustained drone with slow shimmer, ominous and ancient" + ), + "negative": NEG_NOISE, + "duration": 10.0, + }, + # --- SFX (короткие, без лупа) --- + "sfx_step": { + "positive": ( + "single soft footstep on dry crumbly ash and dead grass, short " + "subtle crunch, close up, dry foley" + ), + "negative": NEG_NOISE, + "duration": 2.5, + }, + "sfx_click": { + "positive": ( + "single short UI click, soft wooden tap, clean and dry, " + "interface sound" + ), + "negative": NEG_NOISE, + "duration": 1.5, + }, + "sfx_bell": { + "positive": ( + "single bronze bell strike, resonant metallic ring with long " + "natural decay, solemn, temple bell, one hit only" + ), + "negative": NEG_NOISE, + "duration": 5.0, + }, + "sfx_whoosh": { + "positive": ( + "single quick air whoosh, fast swing passing by, short swoosh " + "transition" + ), + "negative": NEG_NOISE, + "duration": 2.0, + }, + "sfx_hit": { + "positive": ( + "single dull impact hit, heavy blunt thud with brief echo, " + "combat punch sound, one hit only" + ), + "negative": NEG_NOISE, + "duration": 2.0, + }, + "sfx_pickup": { + "positive": ( + "single gentle collect chime, soft bright pling, short " + "delicate pickup sound, one note only" + ), + "negative": NEG_NOISE, + "duration": 2.0, + }, + "sfx_drop": { + "positive": ( + "single water drop into calm pond, soft plop with tiny ripple, " + "close up, one drop only" + ), + "negative": NEG_NOISE, + "duration": 2.0, }, } -DURATION = 10.0 # с — заметно длиннее процедурного лупа (8 с), до crossfade -NUM_INFERENCE_STEPS = 100 # 200 — дефолт качества; на CPU берём 100 для PoC -GUIDANCE = 3.0 +DEFAULT_SEED = 1 -def main() -> None: - if len(sys.argv) < 2 or sys.argv[1] not in PROMPTS: - print(f"использование: gen_audioldm2.py <{'|'.join(PROMPTS)}> [seed]") - sys.exit(1) - name = sys.argv[1] - seed = int(sys.argv[2]) if len(sys.argv) > 2 else 1 - spec = PROMPTS[name] - - import diffusers # noqa: F401 (после разбора аргументов — чтобы --help был быстрым) - - torch.manual_seed(seed) - np.random.seed(seed) - +def load_pipeline(): from diffusers import AudioLDM2Pipeline - # GPU (RTX 3090) при наличии: fp16, иначе CPU fp32. Seed-детерминизм — - # по-устройству: fp16-GPU и fp32-CPU дают разные реализации одного seed. use_gpu = torch.cuda.is_available() dtype = torch.float16 if use_gpu else torch.float32 pipe = AudioLDM2Pipeline.from_pretrained("cvssp/audioldm2", torch_dtype=dtype) pipe.to("cuda" if use_gpu else "cpu") - print(f"устройство: {'cuda fp16' if use_gpu else 'cpu fp32 (12 потоков)'}") if not use_gpu: torch.set_num_threads(12) + return pipe, use_gpu + +def main() -> None: + # Аргументы — id[:seed]; без аргументов — весь LIST с DEFAULT_SEED. + jobs: list[tuple[str, int]] = [] + for arg in sys.argv[1:]: + ident, _, seed = arg.partition(":") + if ident not in LIST: + print(f"неизвестный клип {ident!r}; доступно: {', '.join(LIST)}") + sys.exit(1) + jobs.append((ident, int(seed) if seed else DEFAULT_SEED)) + if not jobs: + jobs = [(name, DEFAULT_SEED) for name in LIST] + + import diffusers # noqa: F401 (после разбора аргументов — чтобы --help был быстрым) + + pipe, use_gpu = load_pipeline() + dev = "cuda fp16" if use_gpu else "cpu fp32" REVIEW.mkdir(parents=True, exist_ok=True) - out = REVIEW / f"{name}_{seed}.wav" - print(f"генерация {name} (seed={seed}, {DURATION} с, шаги {NUM_INFERENCE_STEPS})...") - audio = pipe( - spec["positive"], - negative_prompt=spec["negative"], - num_inference_steps=NUM_INFERENCE_STEPS, - guidance_scale=GUIDANCE, - audio_length_in_s=DURATION, - ).audios[0] + for ident, seed in jobs: + spec = LIST[ident] + torch.manual_seed(seed) + np.random.seed(seed) + print(f"генерация {ident} (seed={seed}, {spec['duration']} с, {dev})...") + audio = pipe( + spec["positive"], + negative_prompt=spec["negative"], + num_inference_steps=100, + guidance_scale=3.0, + audio_length_in_s=spec["duration"], + ).audios[0] + audio = np.clip(audio, -1.0, 1.0) + pcm = (audio * 32767.0).astype(np.int16) + import wave - # AudioLDM2 отдаёт float [-1..1] 16000 Гц моно; пишем PCM16 без клипа. - audio = np.clip(audio, -1.0, 1.0) - pcm = (audio * 32767.0).astype(np.int16) - import wave - - with wave.open(str(out), "wb") as w: - w.setnchannels(1) - w.setsampwidth(2) - w.setframerate(16000) - w.writeframes(pcm.tobytes()) - print(f"готово: {out.relative_to(HERE)} ({pcm.shape[0] / 16000:.1f} с, 16 кГц)") + out = REVIEW / f"{ident}_{seed}.wav" + with wave.open(str(out), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(16000) + w.writeframes(pcm.tobytes()) + print(f"готово: {out.relative_to(HERE)} ({pcm.shape[0] / 16000:.1f} с)") if __name__ == "__main__": diff --git a/apps/game/tools/aiaudio/manifest.json b/apps/game/tools/aiaudio/manifest.json index d7aef3f..520378a 100644 --- a/apps/game/tools/aiaudio/manifest.json +++ b/apps/game/tools/aiaudio/manifest.json @@ -9,9 +9,22 @@ "loop_check": "пост.py --check: скачок на шве против максимума межсэмплового diff в середине записи", "criterion": "амбиент прудов лучше процедурного gen.mjs на слух и формат сходится (22050 моно 16-bit, луп без щелчка)", "items": [ - { "file": "amb_ponds_1.wav", "prompt": "ponds.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "GPU fp16" }, - { "file": "amb_ponds_2.wav", "prompt": "ponds.positive", "seed": 2, "steps": 100, "gate": "pending", "match": "?", "note": "GPU fp16" }, - { "file": "amb_zvenets_1.wav", "prompt": "zvenets.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "GPU fp16" }, - { "file": "amb_zvenets_2.wav", "prompt": "zvenets.positive", "seed": 2, "steps": 100, "gate": "pending", "match": "?", "note": "GPU fp16" } + { "file": "amb_ponds_1.wav", "prompt": "amb_ponds.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "луп; эталон gen.mjs ambiencePonds" }, + { "file": "amb_ponds_2.wav", "prompt": "amb_ponds.positive", "seed": 2, "steps": 100, "gate": "pending", "match": "?", "note": "луп" }, + { "file": "amb_zvenets_1.wav", "prompt": "amb_zvenets.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "луп; эталон ambienceZvenets" }, + { "file": "amb_zvenets_2.wav", "prompt": "amb_zvenets.positive", "seed": 2, "steps": 100, "gate": "pending", "match": "?", "note": "луп" }, + { "file": "amb_meadows_1.wav", "prompt": "amb_meadows.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "луп; в игре луга без амбиента (тишина пепла)" }, + { "file": "amb_meadows_2.wav", "prompt": "amb_meadows.positive", "seed": 2, "steps": 100, "gate": "pending", "match": "?", "note": "луп" }, + { "file": "amb_ponds_night_1.wav", "prompt": "amb_ponds_night.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "луп; эталон ambiencePondsNight" }, + { "file": "amb_ponds_night_2.wav", "prompt": "amb_ponds_night.positive", "seed": 2, "steps": 100, "gate": "pending", "match": "?", "note": "луп" }, + { "file": "amb_tower_hum_1.wav", "prompt": "amb_tower_hum.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "луп; слой layerTowerHum" }, + { "file": "sfx_step_1.wav", "prompt": "sfx_step.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "SFX; в игре спек-синтез" }, + { "file": "sfx_click_1.wav", "prompt": "sfx_click.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "SFX" }, + { "file": "sfx_bell_1.wav", "prompt": "sfx_bell.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "SFX; эталон bellHit" }, + { "file": "sfx_bell_2.wav", "prompt": "sfx_bell.positive", "seed": 2, "steps": 100, "gate": "pending", "match": "?", "note": "SFX" }, + { "file": "sfx_whoosh_1.wav", "prompt": "sfx_whoosh.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "SFX; эталон whoosh" }, + { "file": "sfx_hit_1.wav", "prompt": "sfx_hit.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "SFX" }, + { "file": "sfx_pickup_1.wav", "prompt": "sfx_pickup.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "SFX" }, + { "file": "sfx_drop_1.wav", "prompt": "sfx_drop.positive", "seed": 1, "steps": 100, "gate": "pending", "match": "?", "note": "SFX; эталон капли ambiencePonds" } ] } \ No newline at end of file diff --git a/apps/game/tools/aiaudio/post.py b/apps/game/tools/aiaudio/post.py index 493f7e4..75c4cd3 100644 --- a/apps/game/tools/aiaudio/post.py +++ b/apps/game/tools/aiaudio/post.py @@ -4,8 +4,9 @@ Выход: review/__loop.wav — 22050 Гц, моно 16-бит, нормализованный, бесшовный луп (хвост кроссфейдом вплетается в начало — стык без щелчка). -Запуск: ~/.cache/rpg-ai/venv-pxgpt/bin/python tools/aiaudio/post.py review/amb_ponds_1.wav -Проверка стыка: --check прогоняет сшивку twice и ищет скачок на границе. +Запуск: ~/.cache/rpg-ai/venv-aiaudio/bin/python tools/aiaudio/post.py review/amb_ponds_1.wav +Флаги: --check — сверить стык с фоном записи; --noloop — SFX без лупа + (ресэмпл + нормализация, хвост не вплетается). """ import sys import wave @@ -52,17 +53,20 @@ def main() -> None: args = [a for a in sys.argv[1:] if not a.startswith("--")] check = "--check" in sys.argv + noloop = "--noloop" in sys.argv if len(args) != 1: - print("использование: post.py [--check]") + print("использование: post.py [--check] [--noloop]") sys.exit(1) src = Path(args[0]).resolve() s, rate = read_wav(src) s = resample_poly(s, TARGET_RATE, rate) # 16000 -> 22050 (целые множители: 441/320) + s = s if noloop else loopify(s, TARGET_RATE, FADE) + # Нормализация ПОСЛЕ loopify: пик модели часто в фейдах головы/хвоста, + # которые луп вплетает/выбрасывает — иначе фактический пик ниже цели. peak = np.abs(s).max() if peak > 1e-9: s = s * (PEAK / peak) - s = loopify(s, TARGET_RATE, FADE) - out = src.with_name(src.stem + "_loop.wav") + out = src.with_name(src.stem + ("_fx.wav" if noloop else "_loop.wav")) write_wav(out, s) dur = s.shape[0] / TARGET_RATE print(f"готово: {out.relative_to(HERE)} ({dur:.1f} с, {TARGET_RATE} Гц, пик {PEAK})")