ReltivlyObjectv/video-dataset-processing-scripts

1<#
2.SYNOPSIS
3    Tags a video file using huihui_ai/qwen3-vl-abliterated via Ollama (frame-based)
4.DESCRIPTION
5    Extracts evenly spaced frames from a video, sends them to huihui_ai/qwen3-vl-abliterated,
6    and produces sanitized plain-text tags suitable for LTX2 / WAN 2.2.
7.PARAMETER VideoPath
8    Path to the video file
9.PARAMETER Model
10    Ollama model to use (default: huihui_ai/qwen3-vl-abliterated)
11#>
12
13param(
14    [Parameter(Mandatory = $true, Position = 0)]
15    [string]$VideoPath,
16
17    [Parameter(Mandatory = $false)]
18    [string]$Model = "huihui_ai/qwen3-vl-abliterated"
19)
20
21# ------------------------------------------------------------
22# Configuration
23# ------------------------------------------------------------
24
25$OllamaUrl    = "http://localhost:11434/api/chat"
26$TargetFrames = 5
27$MaxWidth     = 256
28
29# ------------------------------------------------------------
30# Validation
31# ------------------------------------------------------------
32
33if (-not (Test-Path -LiteralPath $VideoPath)) {
34    Write-Error "File not found: $VideoPath"
35    exit 1
36}
37
38Write-Host "Processing: $VideoPath" -ForegroundColor Cyan
39
40# ------------------------------------------------------------
41# Load transcription
42# ------------------------------------------------------------
43
44$TranscriptionPath = [System.IO.Path]::ChangeExtension($VideoPath, ".transcription.txt")
45
46if (Test-Path -LiteralPath $TranscriptionPath) {
47    #$AudioText = Get-Content $TranscriptionPath -Raw
48	$AudioText = Get-Content -LiteralPath $TranscriptionPath -Raw
49} else {
50    $AudioText = ""
51}
52
53# ------------------------------------------------------------
54# Get video duration via ffprobe (robust, machine-readable)
55# ------------------------------------------------------------
56
57$DurationRaw = & ffprobe `
58    -v error `
59    -show_entries format=duration `
60    -of default=noprint_wrappers=1:nokey=1 `
61    $VideoPath
62
63if (-not $DurationRaw) {
64    Write-Error "Unable to determine video duration via ffprobe."
65    exit 1
66}
67
68try {
69    $TotalSeconds = [double]::Parse(
70        $DurationRaw.Trim(),
71        [System.Globalization.CultureInfo]::InvariantCulture
72    )
73} catch {
74    Write-Error "Failed to parse duration returned by ffprobe."
75    exit 1
76}
77
78if ($TotalSeconds -le 0) {
79    Write-Error "Invalid duration returned by ffprobe."
80    exit 1
81}
82
83# ------------------------------------------------------------
84# Compute FPS for even sampling
85# ------------------------------------------------------------
86
87$FPS = $TargetFrames / $TotalSeconds
88
89if ($FPS -le 0) {
90    Write-Error "Computed FPS is invalid."
91    exit 1
92}
93
94# ------------------------------------------------------------
95# Extract frames using ffmpeg (NO manual quoting)
96# ------------------------------------------------------------
97
98$TempDir = Join-Path $env:TEMP ("frames_" + [guid]::NewGuid().ToString())
99New-Item -ItemType Directory -Path $TempDir | Out-Null
100
101$FramePattern = Join-Path $TempDir "frame_%02d.jpg"
102
103& ffmpeg `
104    -hide_banner -loglevel error `
105    -i "$VideoPath" `
106    -vf "fps=$FPS,scale=${MaxWidth}:-1" `
107    -frames:v $TargetFrames `
108    $FramePattern
109
110$FrameFiles = Get-ChildItem $TempDir -Filter "*.jpg" | Sort-Object Name
111
112if ($FrameFiles.Count -ne $TargetFrames) {
113    Write-Error "Frame extraction failed. Expected $TargetFrames frames, got $($FrameFiles.Count)."
114    Remove-Item $TempDir -Recurse -Force
115    exit 1
116}
117
118# ------------------------------------------------------------
119# Base64 encode frames
120# ------------------------------------------------------------
121
122$ImagesBase64 = foreach ($Frame in $FrameFiles) {
123    [Convert]::ToBase64String([IO.File]::ReadAllBytes($Frame.FullName))
124}
125
126# ------------------------------------------------------------
127# Prompt (plain-text only, dataset-safe)
128# ------------------------------------------------------------
129
130$Prompt = @"
131Analyze the short (few seconds) video represented by these sequential frames.
132
133Generate a detailed narrative caption describing the action and scene **visible in the provided frames**.
134
135Return ONE single line of output consisting ONLY of detailed natural language sentences that are each separated by a comma or a period. Lesser details of the same concept are separated by comma and different major concepts are separated by a period.
136Do not use newlines, bullet points, numbering, or JSON.
137Do not include explanations or prose sentences.
138
139Rules:
140- Every element of output must be a descriptive sentence in natural language.
141- Write sentences that describe the specific movements, positions, and interactions between subjects as they happen, using proper terminology to describe any common actions being performed.
142- Use third-person descriptive language (e.g., "a man in a green sweatsuit is speaking to a woman in a pink coat").
143- Focus primarily on the temporal progression of the scene rather than static attributes.
144- Intentionally include descriptions of both the general camera angle and also how the camera is positioned relative to the human subjects (e.g. low angle shot, extreme close up, wide angle shot, medium shot, etc.).
145- include brief descriptions of subject attributes (muscular, overweight, petite, curvy, busty, small breasts, white, blonde, etc.), general age descriptor (such as 20s, mature, elderly, etc), gender presentation, body type, clothing details, and elements that appear mid-video (e.g. the camera pans towards a blonde woman in a green tube top).
146- Strictly limit your description to the actions and scenes visible in the input frames.
147- Do NOT predict, infer, or speculate about future actions.
148- Do NOT continue the scene past the last frame that is shown.
149- Do NOT describe things that you do not see (e.g. if you can't see someone's torso at all, do not specify what their shirt looks like or whether they are even wearing a shirt).
150- Do NOT refer to the media file itself or individual frames; do NOT use phrases like "this image," "this video," "the first frame," etc. Only describe the actions that are visible or result in the shown temporal changes.
151- The videos have already had text transcription performed. In the below block, the audio data is provided; treat this as reference data and include any spoken words in your scene description with double quotations and attribute it to the correct speaker based on the order of who is speaking in the frames. General descriptions in parenthesis or similar like "(explosion)" should be included in the general description rather than explicitly quoted. Spoken words should always be quoted (e.g. the man on the left says "hello", the woman in the pink top asks "how are you?", etc.)
152- Do NOT exclude the double quotations; they should ALWAYS encapsulate dialogue
153
154>>> AUDIO BLOCK START <<<
155__AUDIOPLACEHOLDER___
156>>> AUDIO BLOCK END <<<
157
158Formatting constraints:
159- Output must be one continuous line of sentences only.
160- Maximize descriptive coverage and detail; do not output short tags or single words.
161- Use natural phrasing and slang to describe the actions realistically.
162- This output is intended for LTX-2 training data, which requires descriptive captions rather than short keyword tags.
163"@
164
165# Insert audio transcription into prompt
166$Prompt = $Prompt.Replace("__AUDIOPLACEHOLDER___", $AudioText)
167
168# ------------------------------------------------------------
169# Build Ollama request
170# ------------------------------------------------------------
171
172$RequestBody = @{
173    model = $Model
174    messages = @(
175        @{
176            role    = "user"
177            content = $Prompt
178            images  = $ImagesBase64
179        }
180    )
181    stream = $false
182} | ConvertTo-Json -Depth 10
183
184# ------------------------------------------------------------
185# Send request to Ollama
186# ------------------------------------------------------------
187
188try {
189    Write-Host "Sending frames to Ollama ($Model)..." -ForegroundColor Yellow
190
191    $Response = Invoke-RestMethod `
192        -Uri $OllamaUrl `
193        -Method Post `
194        -Body $RequestBody `
195        -ContentType "application/json" `
196        -TimeoutSec 300
197
198    $RawOutput = $Response.message.content
199} catch {
200    Write-Error "Ollama request failed: $_"
201    Remove-Item $TempDir -Recurse -Force
202    exit 1
203}
204
205# ------------------------------------------------------------
206# Sanitize output for dataset tagging
207# ------------------------------------------------------------
208
209$Sanitized = $RawOutput.ToLower()
210$Sanitized = $Sanitized -replace '[\r\n]', ' '
211#$Sanitized = $Sanitized -replace '[^a-z0-9,." ]', ''
212$Sanitized = $Sanitized -replace '\s+', ' '
213$Sanitized = $Sanitized.Trim(' ,.')
214
215# ------------------------------------------------------------
216# Save output
217# ------------------------------------------------------------
218
219$OutputPath = [System.IO.Path]::ChangeExtension($VideoPath, ".txt")
220$Sanitized | Out-File -FilePath $OutputPath -Encoding UTF8
221
222Write-Host "Tags generated:" -ForegroundColor Green
223Write-Host $Sanitized
224Write-Host "Saved to: $OutputPath" -ForegroundColor DarkGray
225
226# ------------------------------------------------------------
227# Cleanup
228# ------------------------------------------------------------
229
230Remove-Item $TempDir -Recurse -Force