v good
Some checks failed
Deploy to Pages / build (pull_request) Failing after 1m56s

This commit is contained in:
Andras Schmelczer 2026-05-16 13:46:19 +01:00
commit 10a81ba474
45 changed files with 1974 additions and 490 deletions

View file

@ -1,4 +1,4 @@
import { appConfig } from '../config';
import type { GardenAudioEngineConfig } from '../config';
import { clamp, clamp01 } from '../utils/clamp';
import { VibePreset } from '../vibes';
import {
@ -6,7 +6,11 @@ import {
GardenAudioConfig,
GardenAudioVibeProfile,
} from './garden-audio-config';
import { degreeToSemitone, getChordIntervals, getVibeProfile } from './garden-audio-music';
import {
degreeToSemitone,
getChordIntervals,
getVibeProfile,
} from './garden-audio-music';
import { GardenAudioColorIndex, PianoNote } from './garden-audio-types';
interface RenderLookaheadRequest {
@ -23,6 +27,13 @@ interface StrokeAccentRequest {
activity: number;
selectedColorIndex: GardenAudioColorIndex;
mirrorAmount?: number;
panBias?: number;
registerBias?: number;
brightnessBias?: number;
contour?: number;
pressureAmount?: number;
pressureDelta?: number;
maniaAmount?: number;
}
interface TouchDownRequest {
@ -31,6 +42,13 @@ interface TouchDownRequest {
strength: number;
selectedColorIndex: GardenAudioColorIndex;
mirrorAmount?: number;
panBias?: number;
registerBias?: number;
brightnessBias?: number;
contour?: number;
pressureAmount?: number;
pressureDelta?: number;
maniaAmount?: number;
}
interface Register {
@ -61,6 +79,14 @@ interface BrushPhraseLayer {
selectedColorIndex: GardenAudioColorIndex;
energy: number;
mirrorAmount: number;
motifOffsets: Array<number>;
panBias: number;
registerBias: number;
brightnessBias: number;
contour: number;
pressureAmount: number;
pressureDelta: number;
maniaAmount: number;
}
const COLOR_POOLS: [ColorPool, ColorPool, ColorPool] = [
@ -134,6 +160,9 @@ const BRUSH_STREAM_IDLE_INTERVAL_BEATS = 2;
const BRUSH_STREAM_ACTIVE_INTERVAL_BEATS = 1;
const BRUSH_STREAM_INTENSE_INTERVAL_BEATS = 0.5;
const BRUSH_STREAM_MANIC_INTERVAL_BEATS = 0.25;
const BRUSH_MOTIF_MAX_STEPS = 8;
const BRUSH_MOTIF_CANON_DELAY_SECONDS = 0.055;
const PAD_DURATION_BAR_SCALE = 0.46;
export class GenerativePianoEngine {
private nextBeatAt: number | null = null;
@ -154,22 +183,23 @@ export class GenerativePianoEngine {
public constructor(
private readonly config: GardenAudioConfig,
private readonly engineConfig: GardenAudioEngineConfig,
private readonly playNote: (note: PianoNote) => void
) {}
public prime(now: number): void {
if (this.nextBeatAt === null) {
this.nextBeatAt = now + appConfig.audioEngine.startDelaySeconds;
this.nextBeatAt = now + this.engineConfig.startDelaySeconds;
}
this.timelineStartedAt ??= now;
this.nextBrushStreamAt ??= now + appConfig.audioEngine.startDelaySeconds;
this.nextBrushStreamAt ??= now + this.engineConfig.startDelaySeconds;
}
public cue(now: number): void {
this.nextBeatAt = now + appConfig.audioEngine.startDelaySeconds;
this.nextBeatAt = now + this.engineConfig.startDelaySeconds;
this.timelineStartedAt = now;
this.beatIndex = 0;
this.nextBrushStreamAt = now + appConfig.audioEngine.startDelaySeconds;
this.nextBrushStreamAt = now + this.engineConfig.startDelaySeconds;
this.brushStreamNoteIndex = 0;
this.lastBrushStreamMidi = null;
}
@ -188,9 +218,25 @@ export class GenerativePianoEngine {
strength,
selectedColorIndex,
mirrorAmount = 0,
panBias = 0,
registerBias = 0,
brightnessBias = 0.5,
contour = 0,
pressureAmount = 0,
pressureDelta = 0,
maniaAmount = 0,
}: TouchDownRequest): void {
const normalizedStrength = clamp01(strength);
const normalizedMirrorAmount = clamp01(mirrorAmount);
const normalizedMotif = this.normalizeMotif({
panBias,
registerBias,
brightnessBias,
contour,
pressureAmount,
pressureDelta,
maniaAmount,
});
this.isWaitingForGestureAccent = false;
this.lastGestureAccentAt = now;
@ -201,8 +247,17 @@ export class GenerativePianoEngine {
strength: normalizedStrength,
selectedColorIndex,
mirrorAmount: normalizedMirrorAmount,
...normalizedMotif,
});
this.playTouchNote({
vibe,
now,
selectedColorIndex,
strength: normalizedStrength,
panBias: normalizedMotif.panBias,
registerBias: normalizedMotif.registerBias,
brightnessBias: normalizedMotif.brightnessBias,
});
this.playTouchNote(vibe, now, selectedColorIndex, normalizedStrength);
}
public recordStroke({
@ -211,9 +266,25 @@ export class GenerativePianoEngine {
activity,
selectedColorIndex,
mirrorAmount = 0,
panBias = 0,
registerBias = 0,
brightnessBias = 0.5,
contour = 0,
pressureAmount = 0,
pressureDelta = 0,
maniaAmount = 0,
}: StrokeAccentRequest): void {
const strength = clamp01(activity);
const normalizedMirrorAmount = clamp01(mirrorAmount);
const normalizedMotif = this.normalizeMotif({
panBias,
registerBias,
brightnessBias,
contour,
pressureAmount,
pressureDelta,
maniaAmount,
});
if (
this.isWaitingForGestureAccent &&
@ -225,11 +296,19 @@ export class GenerativePianoEngine {
strength,
selectedColorIndex,
mirrorAmount: normalizedMirrorAmount,
...normalizedMotif,
});
return;
}
this.isWaitingForGestureAccent = false;
this.updateBrushPhraseLayer({
now,
strength,
selectedColorIndex,
mirrorAmount: normalizedMirrorAmount,
...normalizedMotif,
});
if (
strength >= STROKE_ACCENT_THRESHOLD &&
now - this.lastStrokeAccentAt >= STROKE_ACCENT_MIN_INTERVAL_SECONDS
@ -385,22 +464,22 @@ export class GenerativePianoEngine {
const chord = this.getChord(profile, barIndex);
const intervals = getChordIntervals(chord, true);
const rootMidi = profile.rootMidi + chord.rootOffset;
const durationSeconds = this.getBarDurationSeconds() * CHORD_BARS * 0.88;
const durationSeconds = this.getBarDurationSeconds() * CHORD_BARS * PAD_DURATION_BAR_SCALE;
const notes = [
{
source: { baseMidi: rootMidi, offsets: [0] },
register: PAD_REGISTERS[0],
velocity: 0.082,
velocity: 0.052,
},
{
source: { baseMidi: rootMidi, offsets: [intervals[1]] },
register: PAD_REGISTERS[1],
velocity: 0.064,
velocity: 0.041,
},
{
source: { baseMidi: rootMidi, offsets: [intervals[3], intervals[2]] },
register: PAD_REGISTERS[2],
velocity: 0.052,
velocity: 0.033,
},
];
@ -412,8 +491,8 @@ export class GenerativePianoEngine {
startTime,
durationSeconds,
pan: register.pan,
delaySend: 0.018,
lowpassHz: this.getLowpassHz(profile, midi, expression * 0.45),
delaySend: 0.008,
lowpassHz: this.getLowpassHz(profile, midi, expression * 0.28),
});
});
}
@ -519,7 +598,7 @@ export class GenerativePianoEngine {
this.config.colorVoices[selectedColorIndex].velocityMultiplier,
startTime:
now +
appConfig.audioEngine.startDelaySeconds +
this.engineConfig.startDelaySeconds +
index * GESTURE_ACCENT_SPACING_SECONDS,
durationSeconds: 0.48 + strength * 0.22,
pan: this.getColorPan(selectedColorIndex),
@ -529,14 +608,26 @@ export class GenerativePianoEngine {
}
}
private playTouchNote(
vibe: VibePreset,
now: number,
selectedColorIndex: GardenAudioColorIndex,
strength: number
): void {
private playTouchNote({
vibe,
now,
selectedColorIndex,
strength,
panBias,
registerBias,
brightnessBias,
}: {
vibe: VibePreset;
now: number;
selectedColorIndex: GardenAudioColorIndex;
strength: number;
panBias: number;
registerBias: number;
brightnessBias: number;
}): void {
const profile = getVibeProfile(this.config, vibe);
const pool = COLOR_POOLS[selectedColorIndex];
const register = this.getBiasedRegister(pool, registerBias, 0);
const chord = this.getChord(profile, this.getGlobalBarIndex(now));
const chordIntervals = getChordIntervals(chord, false);
const rootMidi = profile.rootMidi + chord.rootOffset;
@ -545,7 +636,7 @@ export class GenerativePianoEngine {
baseMidi: rootMidi,
offsets: this.getSupportOffsets(chordIntervals, selectedColorIndex),
},
pool,
register,
this.lastMidiByColor[selectedColorIndex],
true
);
@ -559,9 +650,13 @@ export class GenerativePianoEngine {
this.config.colorVoices[selectedColorIndex].velocityMultiplier,
startTime: now,
durationSeconds: 0.55 + strength * 0.18,
pan: this.getColorPan(selectedColorIndex),
pan: this.getLayerPan(selectedColorIndex, panBias, 0, 0),
delaySend: 0.006,
lowpassHz: this.getLowpassHz(profile, midi, clamp01(0.45 + strength * 0.45)),
lowpassHz: this.getLowpassHz(
profile,
midi,
clamp01(0.45 + strength * 0.35 + brightnessBias * 0.2)
),
});
}
@ -571,12 +666,26 @@ export class GenerativePianoEngine {
strength,
selectedColorIndex,
mirrorAmount,
panBias,
registerBias,
brightnessBias,
contour,
pressureAmount,
pressureDelta,
maniaAmount,
}: {
vibe: VibePreset;
now: number;
strength: number;
selectedColorIndex: GardenAudioColorIndex;
mirrorAmount: number;
panBias: number;
registerBias: number;
brightnessBias: number;
contour: number;
pressureAmount: number;
pressureDelta: number;
maniaAmount: number;
}): void {
const lifetimeSeconds =
BRUSH_LAYER_BASE_SECONDS +
@ -590,6 +699,18 @@ export class GenerativePianoEngine {
selectedColorIndex,
energy: strength,
mirrorAmount,
motifOffsets: this.getInitialMotifOffsets({
selectedColorIndex,
registerBias,
contour,
}),
panBias,
registerBias,
brightnessBias,
contour,
pressureAmount,
pressureDelta,
maniaAmount,
});
if (this.brushPhraseLayers.length > MAX_BRUSH_PHRASE_LAYERS) {
@ -597,6 +718,55 @@ export class GenerativePianoEngine {
}
}
private updateBrushPhraseLayer({
now,
strength,
selectedColorIndex,
mirrorAmount,
panBias,
registerBias,
brightnessBias,
contour,
pressureAmount,
pressureDelta,
maniaAmount,
}: {
now: number;
strength: number;
selectedColorIndex: GardenAudioColorIndex;
mirrorAmount: number;
panBias: number;
registerBias: number;
brightnessBias: number;
contour: number;
pressureAmount: number;
pressureDelta: number;
maniaAmount: number;
}): void {
const layer = this.brushPhraseLayers[this.brushPhraseLayers.length - 1];
if (!layer || layer.expiresAt <= now) {
return;
}
const followAmount = 0.24 + clamp01(strength) * 0.24;
layer.selectedColorIndex = selectedColorIndex;
layer.energy = Math.max(layer.energy * 0.94, strength);
layer.mirrorAmount = Math.max(layer.mirrorAmount * 0.96, mirrorAmount);
layer.panBias = mix(layer.panBias, panBias, followAmount);
layer.registerBias = mix(layer.registerBias, registerBias, followAmount);
layer.brightnessBias = mix(layer.brightnessBias, brightnessBias, followAmount);
layer.contour = mix(layer.contour, contour, followAmount);
layer.pressureAmount = mix(layer.pressureAmount, pressureAmount, followAmount);
layer.pressureDelta = pressureDelta;
layer.maniaAmount = Math.max(layer.maniaAmount * 0.92, maniaAmount);
layer.motifOffsets.push(
this.getMotifOffset({ registerBias, contour, pressureDelta, strength })
);
if (layer.motifOffsets.length > BRUSH_MOTIF_MAX_STEPS) {
layer.motifOffsets = layer.motifOffsets.slice(-BRUSH_MOTIF_MAX_STEPS);
}
}
private renderBrushPhraseLayers({
vibe,
now,
@ -610,8 +780,8 @@ export class GenerativePianoEngine {
activity: number;
selectedColorIndex: GardenAudioColorIndex;
}): void {
const earliestStart = now + appConfig.audioEngine.piano.scheduleAheadSeconds;
this.nextBrushStreamAt ??= now + appConfig.audioEngine.startDelaySeconds;
const earliestStart = now + this.engineConfig.piano.scheduleAheadSeconds;
this.nextBrushStreamAt ??= now + this.engineConfig.startDelaySeconds;
this.brushPhraseLayers = this.brushPhraseLayers.filter(
(layer) => layer.expiresAt > earliestStart
@ -631,6 +801,7 @@ export class GenerativePianoEngine {
startTime: this.nextBrushStreamAt,
intensity: frame.intensity,
selectedColorIndex: frame.selectedColorIndex ?? selectedColorIndex,
layer: frame.layer,
});
}
this.nextBrushStreamAt += this.getBrushStreamIntervalSeconds(frame.intensity);
@ -643,14 +814,22 @@ export class GenerativePianoEngine {
startTime,
intensity,
selectedColorIndex,
layer,
}: {
vibe: VibePreset;
startTime: number;
intensity: number;
selectedColorIndex: GardenAudioColorIndex;
layer: BrushPhraseLayer | null;
}): void {
const profile = getVibeProfile(this.config, vibe);
const pool = COLOR_POOLS[selectedColorIndex];
const maniaAmount = layer?.maniaAmount ?? clamp01((intensity - 0.82) / 0.18);
const register = this.getBiasedRegister(
pool,
layer?.registerBias ?? 0,
maniaAmount * 0.45
);
const chord = this.getChord(profile, this.getGlobalBarIndex(startTime));
const chordIntervals = getChordIntervals(chord, false);
const rootMidi = profile.rootMidi + chord.rootOffset;
@ -662,12 +841,29 @@ export class GenerativePianoEngine {
}
: {
baseMidi: profile.rootMidi,
offsets: this.rotate(
pool.scaleDegrees,
this.brushStreamNoteIndex + selectedColorIndex
).map((degree) => degreeToSemitone(profile, degree)),
offsets: this.getBrushMotifDegrees({
layer,
pool,
selectedColorIndex,
}).map((degree) => degreeToSemitone(profile, degree)),
};
const midi = this.chooseMidi(source, pool, this.lastBrushStreamMidi, true);
const midi = this.chooseMidi(source, register, this.lastBrushStreamMidi, true);
const pan = this.getLayerPan(
selectedColorIndex,
layer?.panBias ?? 0,
maniaAmount,
layer?.mirrorAmount ?? 0
);
const durationSeconds = clamp(
0.48 + intensity * 0.08 - maniaAmount * 0.34,
0.14,
0.62
);
const delaySend = clamp(
0.012 + intensity * 0.011 + (layer?.mirrorAmount ?? 0) * 0.004 - maniaAmount * 0.006,
0.006,
0.032
);
this.lastBrushStreamMidi = midi;
this.lastMidiByColor[selectedColorIndex] = midi;
@ -677,11 +873,38 @@ export class GenerativePianoEngine {
(0.1 + intensity * 0.13) *
this.config.colorVoices[selectedColorIndex].velocityMultiplier,
startTime,
durationSeconds: 0.42 + intensity * 0.22,
pan: this.getColorPan(selectedColorIndex),
delaySend: 0.012 + intensity * 0.01,
lowpassHz: this.getLowpassHz(profile, midi, clamp01(0.35 + intensity * 0.65)),
durationSeconds,
pan,
delaySend,
lowpassHz: this.getLowpassHz(
profile,
midi,
clamp01(
0.32 +
intensity * 0.48 +
(layer?.brightnessBias ?? 0.5) * 0.14 +
maniaAmount * 0.18
)
),
});
if (maniaAmount >= 0.62 && (this.brushStreamNoteIndex % 2 === 1 || intensity >= 0.9)) {
const echoMidi = midi + 12 <= 88 ? midi + 12 : midi - 12;
this.playNote({
midi: echoMidi,
velocity:
(0.045 + intensity * 0.05) *
this.config.colorVoices[selectedColorIndex].velocityMultiplier,
startTime:
startTime +
BRUSH_MOTIF_CANON_DELAY_SECONDS +
(layer?.mirrorAmount ?? 0) * 0.04,
durationSeconds: Math.max(0.11, durationSeconds * 0.68),
pan: clamp(-pan * 0.75, -1, 1),
delaySend: Math.max(0.006, delaySend * 0.72),
lowpassHz: this.getLowpassHz(profile, echoMidi, 0.62 + maniaAmount * 0.24),
});
}
}
private getBrushStreamFrame(
@ -690,17 +913,19 @@ export class GenerativePianoEngine {
): {
intensity: number;
selectedColorIndex: GardenAudioColorIndex | null;
layer: BrushPhraseLayer | null;
} {
const layerStates = this.brushPhraseLayers.map((layer) => ({
layer,
intensity:
layer.energy *
this.getBrushPhraseFade(layer, startTime) *
(0.8 + layer.mirrorAmount * 0.45),
(0.8 + layer.mirrorAmount * 0.45 + layer.maniaAmount * 0.42),
}));
const dominant = layerStates.reduce<
{ layer: BrushPhraseLayer; intensity: number } | null
>((best, state) => {
const dominant = layerStates.reduce<{
layer: BrushPhraseLayer;
intensity: number;
} | null>((best, state) => {
if (state.intensity <= 0) {
return best;
}
@ -712,8 +937,11 @@ export class GenerativePianoEngine {
);
return {
intensity: clamp01(activity * 0.45 + layeredIntensity),
intensity: clamp01(
activity * 0.42 + layeredIntensity + (dominant?.layer.maniaAmount ?? 0) * 0.18
),
selectedColorIndex: dominant?.layer.selectedColorIndex ?? null,
layer: dominant?.layer ?? null,
};
}
@ -735,6 +963,142 @@ export class GenerativePianoEngine {
return clamp01(1 - ageSeconds / Math.max(0.001, lifetimeSeconds));
}
private normalizeMotif({
panBias,
registerBias,
brightnessBias,
contour,
pressureAmount,
pressureDelta,
maniaAmount,
}: {
panBias: number;
registerBias: number;
brightnessBias: number;
contour: number;
pressureAmount: number;
pressureDelta: number;
maniaAmount: number;
}): {
panBias: number;
registerBias: number;
brightnessBias: number;
contour: number;
pressureAmount: number;
pressureDelta: number;
maniaAmount: number;
} {
return {
panBias: clamp(panBias, -1, 1),
registerBias: clamp(registerBias, -1, 1),
brightnessBias: clamp01(brightnessBias),
contour: clamp(contour, -1, 1),
pressureAmount: clamp01(pressureAmount),
pressureDelta: clamp(pressureDelta, -1, 1),
maniaAmount: clamp01(maniaAmount),
};
}
private getInitialMotifOffsets({
selectedColorIndex,
registerBias,
contour,
}: {
selectedColorIndex: GardenAudioColorIndex;
registerBias: number;
contour: number;
}): Array<number> {
const start = selectedColorIndex - 1 + Math.round(registerBias);
const motion = contour > 0.2 ? 1 : contour < -0.2 ? -1 : 0;
return [start, start + motion, start + motion * 2, start + motion];
}
private getMotifOffset({
registerBias,
contour,
pressureDelta,
strength,
}: {
registerBias: number;
contour: number;
pressureDelta: number;
strength: number;
}): number {
const contourStep = contour > 0.3 ? 1 : contour < -0.3 ? -1 : 0;
const registerStep = Math.round(registerBias * 2);
const pressureStep = pressureDelta > 0.08 ? 1 : pressureDelta < -0.08 ? -1 : 0;
const energyStep = strength >= 0.82 ? 1 : strength >= 0.55 ? 0 : -1;
return clamp(contourStep + registerStep + pressureStep + energyStep, -3, 4);
}
private getBrushMotifDegrees({
layer,
pool,
selectedColorIndex,
}: {
layer: BrushPhraseLayer | null;
pool: ColorPool;
selectedColorIndex: GardenAudioColorIndex;
}): Array<number> {
const colorOffset = this.config.colorVoices[selectedColorIndex].scaleDegreeOffset;
if (!layer || layer.motifOffsets.length === 0) {
return this.rotate(pool.scaleDegrees, this.brushStreamNoteIndex + colorOffset);
}
const motifOffset =
layer.motifOffsets[this.brushStreamNoteIndex % layer.motifOffsets.length];
const contourOffset =
layer.contour > 0.28
? this.brushStreamNoteIndex % 3
: layer.contour < -0.28
? -(this.brushStreamNoteIndex % 3)
: 0;
const pressureLift = layer.pressureAmount > 0.68 ? 1 : 0;
const baseOffset = colorOffset + motifOffset + contourOffset + pressureLift;
return this.rotate(
pool.scaleDegrees.map((degree) => degree + baseOffset),
this.brushStreamNoteIndex
);
}
private getBiasedRegister(
register: Register,
registerBias: number,
maniaAmount: number
): Register {
const shift = Math.round(registerBias * 7 + maniaAmount * 4);
const midiMin = clamp(register.midiMin + shift, 36, 86);
const midiMax = clamp(register.midiMax + shift, midiMin + 4, 91);
return {
midiMin,
midiMax,
preferredMidi: clamp(register.preferredMidi + shift, midiMin, midiMax),
pan: register.pan,
};
}
private getLayerPan(
selectedColorIndex: GardenAudioColorIndex,
panBias: number,
maniaAmount: number,
mirrorAmount: number
): number {
const shimmer =
maniaAmount > 0.4
? Math.sin(this.brushStreamNoteIndex * Math.PI * 0.5) * mirrorAmount * 0.14
: 0;
return clamp(
this.getColorPan(selectedColorIndex) +
panBias * (0.18 + maniaAmount * 0.42) +
shimmer,
-1,
1
);
}
private chooseMidi(
pitchSource: PitchSource,
register: Register,
@ -822,10 +1186,7 @@ export class GenerativePianoEngine {
return [chordIntervals[2], 12, chordIntervals[3], chordIntervals[1] + 12];
}
private getChord(
profile: GardenAudioVibeProfile,
barIndex: number
): GardenAudioChord {
private getChord(profile: GardenAudioVibeProfile, barIndex: number): GardenAudioChord {
const progressionIndex =
Math.floor(barIndex / CHORD_BARS) % profile.progression.length;
return profile.progression[progressionIndex];
@ -852,8 +1213,8 @@ export class GenerativePianoEngine {
return clamp(
this.config.piano.lowpassHz * profile.brightness * (0.58 + expression * 0.32) +
midiLift,
appConfig.audioEngine.piano.lowpassMinHz,
appConfig.audioEngine.piano.lowpassMaxHz
this.engineConfig.piano.lowpassMinHz,
this.engineConfig.piano.lowpassMaxHz
);
}
@ -862,7 +1223,7 @@ export class GenerativePianoEngine {
return;
}
const earliestStart = now + appConfig.audioEngine.piano.scheduleAheadSeconds;
const earliestStart = now + this.engineConfig.piano.scheduleAheadSeconds;
if (this.nextBeatAt >= earliestStart) {
return;
}
@ -898,3 +1259,6 @@ export class GenerativePianoEngine {
return values.map((_, index) => values[(index + offset) % values.length]);
}
}
const mix = (from: number, to: number, amount: number): number =>
from + (to - from) * clamp01(amount);