1param(

2 [string]$ERoot = "",

3 [string]$ExperimentName = "Phase2_ParamSeq_WhisperBase_V4_0001",

4 [string]$ModelName = "openai/whisper-base",

5 [string]$DataProjectRoot = "",

6 [string]$Manifest = "",

7 [string]$AudioRoot = "",

8 [string]$Vocab = "",

9 [string]$ProgramRoot = "",

10 [string]$OutputDir = "",

11 [string]$LogsDir = "",

12 [string]$RunDir = "",

13 [string]$ExportDir = "",

14 [switch]$ResumeFromPauseCkpt,

15 [string]$ResumeFromCheckpoint = "",

16 [switch]$AllowNonERoot,

17 [switch]$DisableAutoResume,

18 [switch]$DisableMemWarn,

19 [double]$MemWarnRamThreshold = 0.90,

20 [double]$MemWarnVramThreshold = 0.90,

21 [int]$MemWarnCheckSteps = 25,

22 [int]$MemWarnRepeatSteps = 200,

23 [int]$PerDeviceTrainBatchSize = 1,

24 [int]$PerDeviceEvalBatchSize = 1,

25 [int]$GradientAccumulationSteps = 32,

26 [bool]$GradientCheckpointing = $true,

27 [double]$LearningRate = 3e-5,

28 [double]$NumTrainEpochs = 3.0,

29 [int]$MaxSteps = -1,

30 [int]$SaveSteps = 250,

31 [int]$EvalSteps = 250,

32 [int]$SaveTotalLimit = 5,

33 [double]$EvalRatio = 0.02,

34 [double]$GpuMemFrac = 0.90,

35 [double]$MaxSeconds = 6.0,

36 [int]$MaxTargetLength = 768,

37 [int]$NumWorkers = 0,

38 [int]$PreextractReportEvery = 1000,

39 [switch]$NoFeatureCache,

40 [switch]$NoPreextractFeatures,

41 [switch]$ValidateOnly,

42 [switch]$DryRun

43)

44

45$ErrorActionPreference = "Stop"

46

47function Ensure-Dir {

48 param([Parameter(Mandatory = $true)][string]$PathValue)

49 try {

50 New-Item -ItemType Directory -Force -Path $PathValue -ErrorAction Stop | Out-Null

51 if (-not (Test-Path $PathValue)) {

52 throw "Path still missing after create attempt."

53 }

54 } catch {

55 throw "Failed to ensure directory '$PathValue': $($_.Exception.Message)"

56 }

57}

58

59function Resolve-ExistingPath {

60 param([Parameter(Mandatory = $true)][string]$PathValue)

61 if (-not (Test-Path $PathValue)) {

62 throw "Required path not found: $PathValue"

63 }

64 return (Resolve-Path $PathValue).Path

65}

66

67$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path

68$projectDir = (Resolve-Path (Join-Path $scriptDir "..")).Path

69$mainDir = (Resolve-Path (Join-Path $projectDir "Main")).Path

70$workspaceDir = Split-Path -Parent $projectDir

71

72if (-not $DataProjectRoot) {

73 $localManifest = Join-Path $projectDir "Audio\Samples\Wet_Main\manifest_full.jsonl"

74 $v2ProjectRoot = Join-Path $workspaceDir "AI_Guitar_Pedal_V2"

75 $v2Manifest = Join-Path $v2ProjectRoot "Audio\Samples\Wet_Main\manifest_full.jsonl"

76 if (Test-Path $localManifest) {

77 $DataProjectRoot = $projectDir

78 } elseif (Test-Path $v2Manifest) {

79 $DataProjectRoot = $v2ProjectRoot

80 } else {

81 throw "No complete Phase 2 data tree found. Pass -DataProjectRoot explicitly."

82 }

83}

84$DataProjectRoot = Resolve-ExistingPath -PathValue $DataProjectRoot

85

86if (-not $ERoot) { $ERoot = Join-Path $mainDir "storage" }

87if (-not $Manifest) {

88 $fullManifest = Join-Path $DataProjectRoot "Audio\Samples\Wet_Main\manifest_full.jsonl"

89 if (Test-Path $fullManifest) {

90 $Manifest = $fullManifest

91 } else {

92 $Manifest = Join-Path $DataProjectRoot "Audio\Samples\Wet_Main\manifest.jsonl"

93 }

94}

95if (-not $AudioRoot) { $AudioRoot = Join-Path $DataProjectRoot "Audio\Samples\Wet_Main" }

96if (-not $Vocab) { $Vocab = Join-Path $mainDir "resources\dsp_control_vocab.json" }

97if (-not $ProgramRoot) { $ProgramRoot = Join-Path $DataProjectRoot "Audio\Presets\programs" }

98

99if (-not (Test-Path $ERoot)) {

100 Ensure-Dir -PathValue $ERoot

101}

102$ERoot = (Resolve-Path $ERoot).Path

103$Manifest = Resolve-ExistingPath -PathValue $Manifest

104$AudioRoot = Resolve-ExistingPath -PathValue $AudioRoot

105$Vocab = Resolve-ExistingPath -PathValue $Vocab

106$ProgramRoot = Resolve-ExistingPath -PathValue $ProgramRoot

107

108$dataDir = Join-Path $ERoot "data"

109$artifactsDir = Join-Path $ERoot "artifacts"

110$checkpointsDir = Join-Path $artifactsDir "checkpoints"

111$artifactsLogsDir = Join-Path $artifactsDir "logs"

112$runsDir = Join-Path $artifactsDir "runs"

113$exportsDir = Join-Path $artifactsDir "exports"

114$cacheDir = Join-Path $ERoot "cache"

115$hfCacheDir = Join-Path $cacheDir "hf"

116$transformersCacheDir = Join-Path $cacheDir "transformers"

117$hubCacheDir = Join-Path $cacheDir "hub"

118$datasetsCacheDir = Join-Path $cacheDir "datasets"

119$whisperFeatureCacheDir = Join-Path $cacheDir "whisper\features"

120$torchExtensionsDir = Join-Path $cacheDir "torch\extensions"

121$torchCompileCacheDir = Join-Path $cacheDir "torch\compile_cache"

122$wandbCacheDir = Join-Path $cacheDir "wandb"

123$pipCacheDir = Join-Path $cacheDir "pip"

124$tmpDir = Join-Path $ERoot "tmp"

125

126$mustCreate = @(

127 $dataDir,

128 $artifactsDir,

129 $checkpointsDir,

130 $artifactsLogsDir,

131 $runsDir,

132 $exportsDir,

133 $cacheDir,

134 $hfCacheDir,

135 $transformersCacheDir,

136 $hubCacheDir,

137 $datasetsCacheDir,

138 $whisperFeatureCacheDir,

139 $torchExtensionsDir,

140 $torchCompileCacheDir,

141 $wandbCacheDir,

142 $pipCacheDir,

143 $tmpDir

144)

145foreach ($dirPath in $mustCreate) {

146 Ensure-Dir -PathValue $dirPath

147}

148

149if (-not $OutputDir) { $OutputDir = Join-Path $checkpointsDir $ExperimentName }

150if (-not $LogsDir) { $LogsDir = $artifactsLogsDir }

151if (-not $RunDir) { $RunDir = Join-Path $runsDir $ExperimentName }

152if (-not $ExportDir) { $ExportDir = Join-Path $exportsDir $ExperimentName }

153

154foreach ($dirPath in @($OutputDir, $LogsDir, $RunDir, $ExportDir)) {

155 Ensure-Dir -PathValue $dirPath

156}

157

158$OutputDir = (Resolve-Path $OutputDir).Path

159$LogsDir = (Resolve-Path $LogsDir).Path

160$RunDir = (Resolve-Path $RunDir).Path

161$ExportDir = (Resolve-Path $ExportDir).Path

162

163$env:HF_HOME = $hfCacheDir

164$env:TRANSFORMERS_CACHE = $transformersCacheDir

165$env:HUGGINGFACE_HUB_CACHE = $hubCacheDir

166$env:HF_DATASETS_CACHE = $datasetsCacheDir

167$env:TORCH_EXTENSIONS_DIR = $torchExtensionsDir

168$env:TORCHINDUCTOR_CACHE_DIR = $torchCompileCacheDir

169$env:PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

170$env:XDG_CACHE_HOME = $cacheDir

171$env:TEMP = $tmpDir

172$env:TMP = $tmpDir

173$env:WANDB_DIR = Join-Path $artifactsLogsDir "wandb"

174$env:WANDB_CACHE_DIR = $wandbCacheDir

175$env:TENSORBOARD_LOGDIR = $runsDir

176$env:PIP_CACHE_DIR = $pipCacheDir

177

178Ensure-Dir -PathValue $env:WANDB_DIR

179

180$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"

181$logPrefix = if ($ValidateOnly) { "phase2_validate" } elseif ($DryRun) { "phase2_dry_run" } else { "phase2_full" }

182$logPath = Join-Path $LogsDir ($logPrefix + "_" + $timestamp + ".log")

183

184$allowNonERootValue = if ($AllowNonERoot) { "true" } else { "false" }

185$dryRunValue = if ($DryRun) { "true" } else { "false" }

186$validateOnlyValue = if ($ValidateOnly) { "true" } else { "false" }

187$autoResumeValue = if ($DisableAutoResume) { "false" } else { "true" }

188$memWarnEnabledValue = if ($DisableMemWarn) { "false" } else { "true" }

189$featureCacheValue = if ($NoFeatureCache) { "false" } else { "true" }

190$preextractValue = if ($NoPreextractFeatures) { "false" } else { "true" }

191$gradientCheckpointingValue = if ($GradientCheckpointing) { "true" } else { "false" }

192

193$pythonExe = @(

194 (Join-Path $mainDir ".venv\Scripts\python.exe"),

195 (Join-Path $DataProjectRoot "Main\.venv\Scripts\python.exe")

196) | Where-Object { Test-Path $_ } | Select-Object -First 1

197if (-not $pythonExe) { $pythonExe = "python" }

198

199$args = @(

200 "-u",

201 "train_program_seq2seq.py",

202 "--e_root", $ERoot,

203 "--experiment_name", $ExperimentName,

204 "--model_name", $ModelName,

205 "--allow_non_e_root", $allowNonERootValue,

206 "--dry_run", $dryRunValue,

207 "--validate_only", $validateOnlyValue,

208 "--manifest", $Manifest,

209 "--audio_root", $AudioRoot,

210 "--vocab", $Vocab,

211 "--program_root", $ProgramRoot,

212 "--output_dir", $OutputDir,

213 "--log_dir", $RunDir,

214 "--export_dir", $ExportDir,

215 "--feature_cache_root", $whisperFeatureCacheDir,

216 "--use_feature_cache", $featureCacheValue,

217 "--preextract_features", $preextractValue,

218 "--preextract_report_every", "$PreextractReportEvery",

219 "--max_seconds", "$MaxSeconds",

220 "--max_target_length", "$MaxTargetLength",

221 "--eval_ratio", "$EvalRatio",

222 "--per_device_train_batch_size", "$PerDeviceTrainBatchSize",

223 "--per_device_eval_batch_size", "$PerDeviceEvalBatchSize",

224 "--gradient_accumulation_steps", "$GradientAccumulationSteps",

225 "--learning_rate", "$LearningRate",

226 "--num_train_epochs", "$NumTrainEpochs",

227 "--max_steps", "$MaxSteps",

228 "--gradient_checkpointing", $gradientCheckpointingValue,

229 "--gpu_mem_frac", "$GpuMemFrac",

230 "--num_workers", "$NumWorkers",

231 "--checkpoint_strategy", "steps",

232 "--save_steps", "$SaveSteps",

233 "--eval_steps", "$EvalSteps",

234 "--save_total_limit", "$SaveTotalLimit",

235 "--auto_resume", $autoResumeValue,

236 "--mem_warn_enabled", $memWarnEnabledValue,

237 "--mem_warn_ram_threshold", "$MemWarnRamThreshold",

238 "--mem_warn_vram_threshold", "$MemWarnVramThreshold",

239 "--mem_warn_check_steps", "$MemWarnCheckSteps",

240 "--mem_warn_repeat_steps", "$MemWarnRepeatSteps"

241)

242

243if ($ResumeFromCheckpoint) {

244 $args += @("--resume_from_checkpoint", (Resolve-ExistingPath -PathValue $ResumeFromCheckpoint))

245} elseif ($ResumeFromPauseCkpt) {

246 $pauseCkpt = Join-Path $OutputDir "pause_ckpt"

247 if (Test-Path $pauseCkpt) {

248 $args += @("--resume_from_checkpoint", (Resolve-Path $pauseCkpt).Path)

249 } else {

250 Write-Output "[Warn] pause_ckpt not found, resume skipped: $pauseCkpt"

251 }

252}

253

254$preview = $pythonExe + " " + (($args | ForEach-Object {

255 if ($_ -match "\s") { '"' + $_ + '"' } else { $_ }

256}) -join " ")

257

258Write-Output "[Phase2] Effective paths"

259Write-Output " MainDir=$mainDir"

260Write-Output " DataProjectRoot=$DataProjectRoot"

261Write-Output " Python=$pythonExe"

262Write-Output " ERoot=$ERoot"

263Write-Output " ExperimentName=$ExperimentName"

264Write-Output " ModelName=$ModelName"

265Write-Output " Manifest=$Manifest"

266Write-Output " AudioRoot=$AudioRoot"

267Write-Output " Vocab=$Vocab"

268Write-Output " ProgramRoot=$ProgramRoot"

269Write-Output " OutputDir=$OutputDir"

270Write-Output " LogsDir=$LogsDir"

271Write-Output " RunDir=$RunDir"

272Write-Output " ExportDir=$ExportDir"

273Write-Output " FeatureCacheRoot=$whisperFeatureCacheDir"

274Write-Output " ValidateOnly=$validateOnlyValue"

275Write-Output " DryRun=$dryRunValue"

276Write-Output " MaxSteps=$MaxSteps"

277Write-Output " NumTrainEpochs=$NumTrainEpochs"

278Write-Output " MaxSeconds=$MaxSeconds"

279Write-Output " MaxTargetLength=$MaxTargetLength"

280Write-Output " EvalRatio=$EvalRatio"

281Write-Output " PerDeviceTrainBatchSize=$PerDeviceTrainBatchSize"

282Write-Output " GradientAccumulationSteps=$GradientAccumulationSteps"

283Write-Output " GradientCheckpointing=$gradientCheckpointingValue"

284Write-Output " LearningRate=$LearningRate"

285Write-Output " GpuMemFrac=$GpuMemFrac"

286Write-Output " AutoResume=$autoResumeValue"

287Write-Output " UseFeatureCache=$featureCacheValue"

288Write-Output " PreextractFeatures=$preextractValue"

289Write-Output " TEMP=$env:TEMP"

290Write-Output " HF_HOME=$env:HF_HOME"

291Write-Output " TRANSFORMERS_CACHE=$env:TRANSFORMERS_CACHE"

292Write-Output " HUGGINGFACE_HUB_CACHE=$env:HUGGINGFACE_HUB_CACHE"

293Write-Output " TORCH_EXTENSIONS_DIR=$env:TORCH_EXTENSIONS_DIR"

294Write-Output " TORCHINDUCTOR_CACHE_DIR=$env:TORCHINDUCTOR_CACHE_DIR"

295Write-Output " PYTORCH_CUDA_ALLOC_CONF=$env:PYTORCH_CUDA_ALLOC_CONF"

296Write-Output " WANDB_DIR=$env:WANDB_DIR"

297Write-Output " TENSORBOARD_LOGDIR=$env:TENSORBOARD_LOGDIR"

298Write-Output "[Launch] $preview"

299Write-Output "[Log] $logPath"

300Write-Output "[Hint] Tail log with: Get-Content -LiteralPath `"$logPath`" -Tail 80 -Wait"

301

302Push-Location $mainDir

303try {

304 $oldEap = $ErrorActionPreference

305 $ErrorActionPreference = "Continue"

306 try {

307 & $pythonExe @args 2>&1 | Tee-Object -FilePath $logPath

308 } finally {

309 $ErrorActionPreference = $oldEap

310 }

311 $exitCode = $LASTEXITCODE

312} finally {

313 Pop-Location

314}

315

316Write-Output "[ExitCode] $exitCode"

317Write-Output "[Done] log: $logPath"

318exit $exitCode