This commit is contained in:
Andras Schmelczer 2026-05-17 14:12:01 +01:00
commit 80ed37298b
73 changed files with 3619 additions and 1295 deletions

View file

@ -1,4 +1,5 @@
import { appConfig } from '../config';
import type { PianoNoteRole } from './garden-audio-types';
type GardenAudioChordQuality = 'major' | 'minor';
@ -27,7 +28,154 @@ export interface GardenAudioStylePool extends GardenAudioRegister {
interface GardenAudioGenerativePianoConfig {
stylePools: [GardenAudioStylePool, GardenAudioStylePool, GardenAudioStylePool];
padRegisters: [GardenAudioRegister, GardenAudioRegister, GardenAudioRegister];
chordVoicings: {
majorOpen: Array<number>;
minorOpen: Array<number>;
majorClosed: Array<number>;
minorClosed: Array<number>;
};
vibeChangeStinger: {
velocities: [number, number, number];
pans: [number, number, number];
delaySends: [number, number, number];
lowpassExpression: number;
};
highActivityExtra: {
barOffset: number;
expressionMultiplier: number;
};
padChord: {
velocities: [number, number, number];
expressionVelocityWeight: number;
delaySend: number;
lowpassExpressionWeight: number;
};
supportNote: {
velocityBase: number;
velocityExpressionWeight: number;
durationBaseSeconds: number;
durationExpressionSeconds: number;
delaySendBase: number;
delaySendExpressionWeight: number;
lowpassExpressionWeight: number;
expressionThreshold: number;
offsetsByStyle: [Array<number>, Array<number>, Array<number>];
};
textureNote: {
velocityBase: number;
velocityExpressionWeight: number;
durationBaseSeconds: number;
durationExpressionSeconds: number;
delaySendBase: number;
delaySendExpressionWeight: number;
idleExpressionThreshold: number;
mediumExpressionThreshold: number;
intenseSpacing: number;
idlePhase: number;
};
gestureAccent: {
rotationStrengthMultiplier: number;
quantizeStepLookahead: number;
velocityBase: number;
velocityStrengthWeight: number;
durationBaseSeconds: number;
durationStrengthSeconds: number;
delaySend: number;
};
touchNote: {
registerBiasManiaAmount: number;
velocityBase: number;
velocityStrengthWeight: number;
durationBaseSeconds: number;
durationStrengthSeconds: number;
delaySend: number;
lowpassBaseExpression: number;
lowpassStrengthWeight: number;
};
brushPhrase: {
initialMotifOffset: number;
energyRetain: number;
maniaRetain: number;
energyDecaySeconds: number;
maniaDecaySeconds: number;
fadeMinimumLifetimeSeconds: number;
layerIntensityBase: number;
layerIntensityManiaWeight: number;
frameActivityWeight: number;
frameManiaWeight: number;
};
brushStream: {
inferredManiaThreshold: number;
inferredManiaRange: number;
registerManiaShift: number;
chordToneEverySteps: number;
durationBaseSeconds: number;
durationIntensitySeconds: number;
durationManiaSeconds: number;
durationMinSeconds: number;
durationMaxSeconds: number;
delaySendBase: number;
delaySendIntensityWeight: number;
delaySendManiaWeight: number;
delaySendMin: number;
delaySendMax: number;
velocityBase: number;
velocityIntensityWeight: number;
lowpassBaseExpression: number;
lowpassIntensityWeight: number;
lowpassManiaWeight: number;
manicThreshold: number;
intenseThreshold: number;
activeThreshold: number;
};
brushStreamEcho: {
maniaThreshold: number;
stepModulo: number;
stepRemainder: number;
intensityThreshold: number;
octaveSemitones: number;
maxMidi: number;
velocityBase: number;
velocityIntensityWeight: number;
durationMinSeconds: number;
durationScale: number;
panScale: number;
delaySendMin: number;
delaySendScale: number;
lowpassBaseExpression: number;
lowpassManiaWeight: number;
};
brushMotif: {
highThreshold: number;
mediumThreshold: number;
highOffset: number;
mediumOffset: number;
lowOffset: number;
minOffset: number;
maxOffset: number;
};
registerBias: {
maniaShiftSemitones: number;
midiMin: number;
midiMaxForMin: number;
minimumSpan: number;
midiMax: number;
};
candidateOctaveSearch: {
min: number;
max: number;
};
styleRotationMinSeconds: number;
stylePanOffsetScale: number;
lowpass: {
midiBase: number;
midiRange: number;
midiLiftHz: number;
expressionBase: number;
expressionWeight: number;
};
styleRotationSeconds: number;
styleRotationBars: number;
chordBars: number;
supportBarSpacing: number;
supportBarOffset: number;
@ -38,13 +186,16 @@ interface GardenAudioGenerativePianoConfig {
highActivityExtraThreshold: number;
noteScorePreferenceWeight: number;
noteScoreRegisterWeight: number;
noteScoreChordToneWeight: number;
noteScoreRepeatPenalty: number;
gestureAccentMinIntervalSeconds: number;
strokeAccentMinIntervalSeconds: number;
strokeAccentMinSteps: number;
strokeAccentThreshold: number;
stingerDurationSeconds: number;
stingerSpacingSeconds: number;
maxBrushPhraseLayers: number;
maxBrushStreamNotesPerBar: number;
brushLayerBaseSeconds: number;
brushLayerEnergySeconds: number;
brushLayerMinIntensity: number;
@ -80,10 +231,15 @@ export interface GardenAudioConfig {
feedbackMin: number;
outputActivityWeight: number;
outputBase: number;
outputActivityDuck: number;
timeRampSeconds: number;
feedbackHighPassHz: number;
feedbackLowPassHz: number;
returnLowPassHz: number;
};
piano: {
maxVoices: number;
filterType: BiquadFilterType;
gain: number;
sustainSeconds: number;
sustainLevel: number;
@ -103,6 +259,12 @@ export interface GardenAudioConfig {
tailStopExtraSeconds: number;
voiceStealFadeSeconds: number;
voiceStealStopSeconds: number;
sampleBaseUrl: string;
preloadDecode: {
channels: number;
frames: number;
sampleRateHz: number;
};
};
rhythm: {
bpm: number;
@ -117,6 +279,8 @@ export interface GardenAudioConfig {
filterMinHz: number;
filterMaxHz: number;
durationSeconds: number;
pan: number;
pianoActivity: number;
};
energy: {
attackSeconds: number;
@ -134,12 +298,37 @@ export interface GardenAudioConfig {
noiseMin: number;
unlockTickFrequencyHz: number;
unlockTickSeconds: number;
unlockTickType: OscillatorType;
latencyHint: AudioContextLatencyCategory;
outputFilterType: BiquadFilterType;
noiseBufferChannels: number;
noiseBufferDurationSeconds: number;
pianoBusGains: Record<PianoNoteRole, number>;
pianoBusActivityDucking: Record<PianoNoteRole, number>;
noiseBusGain: number;
compressor: {
thresholdDb: number;
kneeDb: number;
ratio: number;
attackSeconds: number;
releaseSeconds: number;
};
};
input: {
distanceWindowForFullActivityPixels: number;
distanceWindowSeconds: number;
fallbackFrameSeconds: number;
fullActivitySpeed: number;
activityNoiseFloorSpeed: number;
activityCurve: number;
activitySoftCeiling: number;
activityAttackSeconds: number;
activityReleaseSeconds: number;
minAudibleDistance: number;
manicActivityThreshold: number;
manicReleaseThreshold: number;
maniaSmoothingSeconds: number;
minElapsedSeconds: number;
};
muteGain: number;
muteRampSeconds: number;
@ -149,6 +338,7 @@ export interface GardenAudioConfig {
offsetRandomSeconds: number;
scheduleAheadSeconds: number;
silentGain: number;
filterType: BiquadFilterType;
};
startDelaySeconds: number;
vibeChangeStingerMinIntervalSeconds: number;

View file

@ -2,38 +2,94 @@ import { describe, expect, it } from 'vitest';
import { gardenAudioConfig } from './garden-audio-config';
import { GardenAudioGestureState } from './garden-audio-gesture-state';
import type { GardenAudioStrokeMetrics } from './garden-audio-input';
const makeMetrics = ({
elapsedSeconds,
normalizedDistance,
}: {
elapsedSeconds: number;
normalizedDistance: number;
}): GardenAudioStrokeMetrics => ({
distancePixels: normalizedDistance * 1000,
elapsedSeconds,
normalizedDistance,
normalizedSpeed: normalizedDistance / elapsedSeconds,
});
describe('GardenAudioGestureState', () => {
it('uses only distance accumulated in the last half second', () => {
it('ignores tiny jitter below the activity speed floor', () => {
const state = new GardenAudioGestureState(gardenAudioConfig.input);
state.beginGesture();
expect(
state.recordStroke({
metrics: {
distancePixels: 70,
metrics: makeMetrics({
elapsedSeconds: 0.1,
},
}).activity
).toBeCloseTo(0.5);
expect(
state.recordStroke({
metrics: {
distancePixels: 70,
elapsedSeconds: 0.1,
},
}).activity
).toBe(1);
expect(
state.recordStroke({
metrics: {
distancePixels: 0,
elapsedSeconds: 0.51,
},
normalizedDistance: 0.001,
}),
}).activity
).toBe(0);
});
it('normalizes equal drawing speeds across pointer sample rates', () => {
const lowRate = new GardenAudioGestureState(gardenAudioConfig.input);
const highRate = new GardenAudioGestureState(gardenAudioConfig.input);
lowRate.beginGesture();
highRate.beginGesture();
const lowRateFrame = lowRate.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.1,
normalizedDistance: 0.07,
}),
});
let highRateFrame = { activity: 0, maniaAmount: 0 };
for (let index = 0; index < 5; index += 1) {
highRateFrame = highRate.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.02,
normalizedDistance: 0.014,
}),
});
}
expect(highRateFrame.activity).toBeCloseTo(lowRateFrame.activity, 5);
});
it('holds mania with hysteresis before releasing', () => {
const state = new GardenAudioGestureState(gardenAudioConfig.input);
state.beginGesture();
const manicFrame = state.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.25,
normalizedDistance: 0.3,
}),
});
expect(manicFrame.maniaAmount).toBeGreaterThan(0);
const heldFrame = state.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.06,
normalizedDistance: 0.04,
}),
});
expect(heldFrame.maniaAmount).toBeGreaterThan(0);
const releasedFrame = state.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.7,
normalizedDistance: 0,
}),
});
expect(releasedFrame.maniaAmount).toBeLessThan(heldFrame.maniaAmount);
});
});

View file

@ -1,4 +1,4 @@
import { clamp01 } from '../utils/clamp';
import { clamp, clamp01 } from '../utils/clamp';
import type { GardenAudioConfig } from './garden-audio-config';
import type { GardenAudioStrokeMetrics } from './garden-audio-input';
@ -7,14 +7,11 @@ interface GardenAudioGestureFrame {
maniaAmount: number;
}
interface GestureDistanceSample {
at: number;
distancePixels: number;
}
export class GardenAudioGestureState {
private readonly samples: Array<GestureDistanceSample> = [];
private gestureClockSeconds = 0;
private activity = 0;
private maniaAmount = 0;
private isManic = false;
public constructor(private readonly inputConfig: GardenAudioConfig['input']) {}
@ -33,41 +30,79 @@ export class GardenAudioGestureState {
}): GardenAudioGestureFrame {
this.gestureClockSeconds += metrics.elapsedSeconds;
if (metrics.distancePixels > 0) {
this.samples.push({
at: this.gestureClockSeconds,
distancePixels: metrics.distancePixels,
});
}
this.trimSamples();
const windowDistancePixels = this.samples.reduce(
(total, sample) => total + sample.distancePixels,
0
const targetActivity = this.getTargetActivity(metrics);
const activityTimeConstant =
targetActivity > this.activity
? this.inputConfig.activityAttackSeconds
: this.inputConfig.activityReleaseSeconds;
this.activity = approach(
this.activity,
targetActivity,
metrics.elapsedSeconds,
activityTimeConstant
);
const activity = clamp01(
windowDistancePixels / this.inputConfig.distanceWindowForFullActivityPixels
if (this.activity >= this.inputConfig.manicActivityThreshold) {
this.isManic = true;
} else if (this.activity <= this.inputConfig.manicReleaseThreshold) {
this.isManic = false;
}
const maniaTarget = this.isManic
? smoothstep(this.inputConfig.manicReleaseThreshold, 1, this.activity)
: 0;
this.maniaAmount = approach(
this.maniaAmount,
maniaTarget,
metrics.elapsedSeconds,
this.inputConfig.maniaSmoothingSeconds
);
return {
activity,
maniaAmount: smoothstep(this.inputConfig.manicActivityThreshold, 1, activity),
activity: this.activity,
maniaAmount: this.maniaAmount,
};
}
public reset(): void {
this.samples.length = 0;
this.gestureClockSeconds = 0;
this.activity = 0;
this.maniaAmount = 0;
this.isManic = false;
}
private trimSamples(): void {
const earliest = this.gestureClockSeconds - this.inputConfig.distanceWindowSeconds;
while (this.samples.length > 0 && this.samples[0].at < earliest) {
this.samples.shift();
}
private getTargetActivity(metrics: GardenAudioStrokeMetrics): number {
const normalizedSpeed = Math.max(0, metrics.normalizedSpeed);
const activeSpeed = Math.max(
this.inputConfig.activityNoiseFloorSpeed,
this.inputConfig.fullActivitySpeed
);
const speedAmount = clamp01(
(normalizedSpeed - this.inputConfig.activityNoiseFloorSpeed) /
(activeSpeed - this.inputConfig.activityNoiseFloorSpeed)
);
const distanceAmount = clamp01(
metrics.normalizedDistance / this.inputConfig.minAudibleDistance
);
const activity = Math.pow(
speedAmount,
Math.max(0.001, this.inputConfig.activityCurve)
);
return clamp(activity * distanceAmount, 0, this.inputConfig.activitySoftCeiling);
}
}
const approach = (
current: number,
target: number,
elapsedSeconds: number,
timeConstantSeconds: number
): number => {
const amount = 1 - Math.exp(-elapsedSeconds / Math.max(0.001, timeConstantSeconds));
return current + (target - current) * amount;
};
const smoothstep = (edge0: number, edge1: number, value: number): number => {
const amount = clamp01((value - edge0) / (edge1 - edge0));
return amount * amount * (3 - 2 * amount);

View file

@ -0,0 +1,92 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { VIBE_PRESETS } from '../vibes';
import { gardenAudioConfig } from './garden-audio-config';
import { GardenAudioGraph } from './garden-audio-graph';
class FakeAudioParam {
public value = 0;
public setTargetAtTime = vi.fn();
}
class FakeAudioNode {
public readonly gain = new FakeAudioParam();
public readonly frequency = new FakeAudioParam();
public readonly threshold = new FakeAudioParam();
public readonly knee = new FakeAudioParam();
public readonly ratio = new FakeAudioParam();
public readonly attack = new FakeAudioParam();
public readonly release = new FakeAudioParam();
public readonly delayTime = new FakeAudioParam();
public readonly connections: Array<unknown> = [];
public type = '';
public connect(target: unknown): unknown {
this.connections.push(target);
return target;
}
}
class FakeAudioBuffer {
private readonly data: Float32Array;
public constructor(length: number) {
this.data = new Float32Array(length);
}
public getChannelData(): Float32Array {
return this.data;
}
}
class FakeAudioContext {
public readonly currentTime = 1;
public readonly destination = new FakeAudioNode() as unknown as AudioDestinationNode;
public readonly sampleRate = 16;
public readonly state = 'running';
public readonly compressors: Array<FakeAudioNode> = [];
public createGain(): GainNode {
return new FakeAudioNode() as unknown as GainNode;
}
public createBiquadFilter(): BiquadFilterNode {
return new FakeAudioNode() as unknown as BiquadFilterNode;
}
public createDelay(): DelayNode {
return new FakeAudioNode() as unknown as DelayNode;
}
public createDynamicsCompressor(): DynamicsCompressorNode {
const node = new FakeAudioNode();
this.compressors.push(node);
return node as unknown as DynamicsCompressorNode;
}
public createBuffer(_channels: number, length: number): AudioBuffer {
return new FakeAudioBuffer(length) as unknown as AudioBuffer;
}
}
describe('GardenAudioGraph', () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it('builds controlled output, role buses, and delay automation', () => {
vi.stubGlobal('AudioContext', FakeAudioContext);
const graph = new GardenAudioGraph(gardenAudioConfig);
const context = graph.ensureContext(true) as unknown as FakeAudioContext;
expect(context.compressors).toHaveLength(1);
expect(graph.getPianoBus('pad')).not.toBeNull();
expect(graph.getPianoBus('pad')).not.toBe(graph.getPianoBus('gesture'));
expect(graph.noiseBus).not.toBeNull();
graph.updateDelay(VIBE_PRESETS[0].audio, 1);
expect(graph.getPianoBus('pad')?.gain.setTargetAtTime).toHaveBeenCalled();
expect(graph.getPianoBus('brush')?.gain.setTargetAtTime).toHaveBeenCalled();
});
});

View file

@ -1,16 +1,19 @@
import { clamp } from '../utils/clamp';
import { GardenAudioConfig, GardenAudioVibeProfile } from './garden-audio-config';
import type { PianoNoteRole } from './garden-audio-types';
export class GardenAudioGraph {
public context: AudioContext | null = null;
public eventBus: GainNode | null = null;
public delayInput: GainNode | null = null;
public noiseBus: GainNode | null = null;
public noiseBuffer: AudioBuffer | null = null;
private masterGain: GainNode | null = null;
private delayNode: DelayNode | null = null;
private delayFeedback: GainNode | null = null;
private delayOutput: GainNode | null = null;
private readonly pianoBuses = new Map<PianoNoteRole, GainNode>();
public constructor(private readonly config: GardenAudioConfig) {}
@ -30,19 +33,28 @@ export class GardenAudioGraph {
let context: AudioContext;
try {
context = new AudioContextConstructor({ latencyHint: 'interactive' });
context = new AudioContextConstructor({
latencyHint: this.config.graph.latencyHint,
});
} catch {
context = new AudioContextConstructor();
}
const masterGain = context.createGain();
const highPass = context.createBiquadFilter();
const compressor = context.createDynamicsCompressor();
masterGain.gain.value = 0;
highPass.type = 'highpass';
highPass.type = this.config.graph.outputFilterType;
highPass.frequency.value = this.config.highPassFrequencyHz;
compressor.threshold.value = this.config.graph.compressor.thresholdDb;
compressor.knee.value = this.config.graph.compressor.kneeDb;
compressor.ratio.value = this.config.graph.compressor.ratio;
compressor.attack.value = this.config.graph.compressor.attackSeconds;
compressor.release.value = this.config.graph.compressor.releaseSeconds;
masterGain.connect(highPass);
highPass.connect(context.destination);
highPass.connect(compressor);
compressor.connect(context.destination);
this.context = context;
this.masterGain = masterGain;
@ -65,7 +77,7 @@ export class GardenAudioGraph {
const source = this.context.createOscillator();
const gain = this.context.createGain();
source.type = 'sine';
source.type = this.config.graph.unlockTickType;
source.frequency.setValueAtTime(this.config.graph.unlockTickFrequencyHz, now);
gain.gain.setValueAtTime(this.config.piano.minGain, now);
gain.gain.exponentialRampToValueAtTime(
@ -116,6 +128,7 @@ export class GardenAudioGraph {
}
const now = this.context.currentTime;
const normalizedActivity = clamp(activity, 0, 1);
this.delayNode.delayTime.setTargetAtTime(
this.config.delay.timeSeconds * profile.delayTimeMultiplier,
now,
@ -123,7 +136,8 @@ export class GardenAudioGraph {
);
this.delayFeedback.gain.setTargetAtTime(
clamp(
this.config.delay.feedback + activity * this.config.delay.activityFeedbackWeight,
this.config.delay.feedback +
normalizedActivity * this.config.delay.activityFeedbackWeight,
this.config.delay.feedbackMin,
this.config.delay.feedbackMax
),
@ -133,10 +147,16 @@ export class GardenAudioGraph {
this.delayOutput.gain.setTargetAtTime(
this.config.delay.wetGain *
(this.config.delay.outputBase +
activity * this.config.delay.outputActivityWeight),
normalizedActivity * this.config.delay.outputActivityWeight) *
(1 - normalizedActivity * this.config.delay.outputActivityDuck),
now,
this.config.updateRampSeconds
);
this.updatePianoBusGains(normalizedActivity, now);
}
public getPianoBus(role: PianoNoteRole | undefined): GainNode | null {
return this.pianoBuses.get(role ?? 'gesture') ?? this.eventBus;
}
public async close(): Promise<void> {
@ -165,15 +185,27 @@ export class GardenAudioGraph {
const delayNode = context.createDelay(this.config.graph.delayMaxSeconds);
const delayFeedback = context.createGain();
const delayOutput = context.createGain();
const feedbackHighPass = context.createBiquadFilter();
const feedbackLowPass = context.createBiquadFilter();
const returnLowPass = context.createBiquadFilter();
delayNode.delayTime.value = this.config.delay.timeSeconds;
delayFeedback.gain.value = this.config.delay.feedback;
delayOutput.gain.value = this.config.delay.wetGain;
feedbackHighPass.type = 'highpass';
feedbackHighPass.frequency.value = this.config.delay.feedbackHighPassHz;
feedbackLowPass.type = 'lowpass';
feedbackLowPass.frequency.value = this.config.delay.feedbackLowPassHz;
returnLowPass.type = 'lowpass';
returnLowPass.frequency.value = this.config.delay.returnLowPassHz;
delayInput.connect(delayNode);
delayNode.connect(delayFeedback);
delayNode.connect(feedbackHighPass);
feedbackHighPass.connect(feedbackLowPass);
feedbackLowPass.connect(delayFeedback);
delayFeedback.connect(delayNode);
delayNode.connect(delayOutput);
delayNode.connect(returnLowPass);
returnLowPass.connect(delayOutput);
delayOutput.connect(masterGain);
this.delayInput = delayInput;
@ -183,13 +215,50 @@ export class GardenAudioGraph {
}
private createBuses(context: AudioContext, masterGain: GainNode): void {
this.eventBus = context.createGain();
this.eventBus.gain.value = this.config.graph.eventBusGain;
this.eventBus.connect(masterGain);
const eventBus = context.createGain();
eventBus.gain.value = this.config.graph.eventBusGain;
eventBus.connect(masterGain);
this.eventBus = eventBus;
this.pianoBuses.clear();
(Object.keys(this.config.graph.pianoBusGains) as Array<PianoNoteRole>).forEach(
(role) => {
const bus = context.createGain();
bus.gain.value = this.config.graph.pianoBusGains[role];
bus.connect(eventBus);
this.pianoBuses.set(role, bus);
}
);
this.noiseBus = context.createGain();
this.noiseBus.gain.value = this.config.graph.noiseBusGain;
this.noiseBus.connect(eventBus);
}
private updatePianoBusGains(activity: number, now: number): void {
this.pianoBuses.forEach((bus, role) => {
const baseGain = this.config.graph.pianoBusGains[role];
const ducking = this.config.graph.pianoBusActivityDucking[role];
bus.gain.setTargetAtTime(
Math.max(0, baseGain * (1 - activity * ducking)),
now,
this.config.updateRampSeconds
);
});
}
private createNoiseBuffer(context: AudioContext): AudioBuffer {
const buffer = context.createBuffer(1, context.sampleRate, context.sampleRate);
const buffer = context.createBuffer(
appPositiveInteger(this.config.graph.noiseBufferChannels),
Math.max(
1,
Math.floor(
context.sampleRate *
Math.max(0.001, this.config.graph.noiseBufferDurationSeconds)
)
),
context.sampleRate
);
const data = buffer.getChannelData(0);
for (let index = 0; index < data.length; index++) {
@ -205,10 +274,15 @@ export class GardenAudioGraph {
this.context = null;
this.eventBus = null;
this.delayInput = null;
this.noiseBus = null;
this.noiseBuffer = null;
this.masterGain = null;
this.delayNode = null;
this.delayFeedback = null;
this.delayOutput = null;
this.pianoBuses.clear();
}
}
const appPositiveInteger = (value: number): number =>
Math.max(1, Math.floor(Number.isFinite(value) ? value : 1));

View file

@ -0,0 +1,52 @@
import { describe, expect, it } from 'vitest';
import { gardenAudioConfig } from './garden-audio-config';
import { getStrokeMetrics } from './garden-audio-input';
describe('getStrokeMetrics', () => {
it('normalizes stroke distance against canvas size', () => {
const standardDensity = getStrokeMetrics(
{
vibe: {} as never,
from: [0, 0],
to: [100, 0],
canvasSize: [1000, 500],
elapsedSeconds: 0.1,
isErasing: false,
},
gardenAudioConfig.input
);
const highDensity = getStrokeMetrics(
{
vibe: {} as never,
from: [0, 0],
to: [200, 0],
canvasSize: [2000, 1000],
elapsedSeconds: 0.1,
isErasing: false,
},
gardenAudioConfig.input
);
expect(highDensity.normalizedDistance).toBeCloseTo(
standardDensity.normalizedDistance
);
expect(highDensity.normalizedSpeed).toBeCloseTo(standardDensity.normalizedSpeed);
});
it('uses configured elapsed-time floors for missing or invalid samples', () => {
const metrics = getStrokeMetrics(
{
vibe: {} as never,
from: [0, 0],
to: [10, 0],
elapsedSeconds: 0,
isErasing: false,
},
gardenAudioConfig.input
);
expect(metrics.elapsedSeconds).toBe(gardenAudioConfig.input.fallbackFrameSeconds);
expect(Number.isFinite(metrics.normalizedSpeed)).toBe(true);
});
});

View file

@ -4,6 +4,8 @@ import type { GardenAudioStroke } from './garden-audio-types';
export interface GardenAudioStrokeMetrics {
distancePixels: number;
elapsedSeconds: number;
normalizedDistance: number;
normalizedSpeed: number;
}
export const getStrokeMetrics = (
@ -12,10 +14,16 @@ export const getStrokeMetrics = (
): GardenAudioStrokeMetrics => {
const dx = stroke.to[0] - stroke.from[0];
const dy = stroke.to[1] - stroke.from[1];
const distancePixels = Math.hypot(dx, dy);
const elapsedSeconds = getElapsedSeconds(stroke, inputConfig);
const normalizedDistance =
distancePixels / getStrokeNormalizationPixels(stroke, inputConfig);
return {
distancePixels: Math.hypot(dx, dy),
elapsedSeconds: getElapsedSeconds(stroke, inputConfig),
distancePixels,
elapsedSeconds,
normalizedDistance,
normalizedSpeed: normalizedDistance / elapsedSeconds,
};
};
@ -28,8 +36,28 @@ const getElapsedSeconds = (
Number.isFinite(stroke.elapsedSeconds) &&
stroke.elapsedSeconds > 0
) {
return Math.max(0.001, stroke.elapsedSeconds);
return Math.max(inputConfig.minElapsedSeconds, stroke.elapsedSeconds);
}
return inputConfig.fallbackFrameSeconds;
};
const getStrokeNormalizationPixels = (
stroke: GardenAudioStroke,
inputConfig: GardenAudioConfig['input']
): number => {
const width = stroke.canvasSize?.[0];
const height = stroke.canvasSize?.[1];
if (
width !== undefined &&
height !== undefined &&
Number.isFinite(width) &&
Number.isFinite(height) &&
width > 0 &&
height > 0
) {
return Math.max(1, Math.min(width, height));
}
return Math.max(1, inputConfig.distanceWindowForFullActivityPixels);
};

View file

@ -1,5 +1,9 @@
import { VibePreset } from '../vibes';
import { GardenAudioChord, GardenAudioVibeProfile } from './garden-audio-config';
import {
GardenAudioChord,
gardenAudioConfig,
GardenAudioVibeProfile,
} from './garden-audio-config';
export const getVibeProfile = (vibe: VibePreset): GardenAudioVibeProfile => vibe.audio;
@ -8,10 +12,14 @@ export const getChordIntervals = (
openVoicing: boolean
): Array<number> => {
if (openVoicing) {
return chord.quality === 'major' ? [0, 7, 12, 16] : [0, 7, 12, 15];
return chord.quality === 'major'
? gardenAudioConfig.generativePiano.chordVoicings.majorOpen
: gardenAudioConfig.generativePiano.chordVoicings.minorOpen;
}
return chord.quality === 'major' ? [0, 4, 7, 12, 16] : [0, 3, 7, 12, 15];
return chord.quality === 'major'
? gardenAudioConfig.generativePiano.chordVoicings.majorClosed
: gardenAudioConfig.generativePiano.chordVoicings.minorClosed;
};
export const degreeToSemitone = (
@ -21,5 +29,7 @@ export const degreeToSemitone = (
const scaleIndex =
((degree % profile.scale.length) + profile.scale.length) % profile.scale.length;
const octave = Math.floor(degree / profile.scale.length);
return profile.scale[scaleIndex] + octave * 12;
return (
profile.scale[scaleIndex] + octave * gardenAudioConfig.piano.pitchSemitonesPerOctave
);
};

View file

@ -9,6 +9,7 @@ export interface GardenAudioStroke {
vibe: VibePreset;
from: ArrayLike<number>;
to: ArrayLike<number>;
canvasSize?: ArrayLike<number>;
isErasing: boolean;
elapsedSeconds?: number;
}
@ -34,10 +35,19 @@ export interface PianoNote {
startTime: number;
durationSeconds: number;
pan: number;
role?: PianoNoteRole;
delaySend?: number;
lowpassHz?: number;
}
export type PianoNoteRole =
| 'pad'
| 'support'
| 'texture'
| 'gesture'
| 'brush'
| 'stinger';
export interface NoiseBurst {
startTime: number;
durationSeconds: number;

View file

@ -14,6 +14,7 @@ const calls = {
resumed: 0,
sourcesStarted: 0,
sources: [] as Array<FakeScheduledSourceNode>,
gains: [] as Array<FakeAudioNode>,
};
let contextState: AudioContextState = 'suspended';
@ -80,7 +81,9 @@ class FakeAudioContext {
}
public createGain(): GainNode {
return new FakeAudioNode() as unknown as GainNode;
const node = new FakeAudioNode();
calls.gains.push(node);
return node as unknown as GainNode;
}
public createBiquadFilter(): BiquadFilterNode {
@ -91,6 +94,10 @@ class FakeAudioContext {
return new FakeAudioNode() as unknown as DelayNode;
}
public createDynamicsCompressor(): DynamicsCompressorNode {
return new FakeAudioNode() as unknown as DynamicsCompressorNode;
}
public createStereoPanner(): StereoPannerNode {
return new FakeAudioNode() as unknown as StereoPannerNode;
}
@ -151,6 +158,7 @@ describe('GardenAudio startup policy', () => {
calls.resumed = 0;
calls.sourcesStarted = 0;
calls.sources = [];
calls.gains = [];
contextState = 'suspended';
resumeError = null;
vi.stubGlobal('AudioContext', FakeAudioContext);
@ -210,6 +218,40 @@ describe('GardenAudio startup policy', () => {
});
});
it('updates live master gain from adjustable volume', () => {
const config = makeConfig();
const audio = new GardenAudio(config);
const vibe = VIBE_PRESETS[0];
audio.start(vibe, { userGesture: true });
const masterGain = calls.gains[0]?.gain;
if (!masterGain) {
throw new Error('Missing fake master gain');
}
audio.setMasterVolume(0.2);
expect(masterGain.setTargetAtTime).toHaveBeenLastCalledWith(
0.2,
1,
config.updateRampSeconds
);
audio.setMuted(true);
const mutedCallCount = masterGain.setTargetAtTime.mock.calls.length;
audio.setMasterVolume(0.8);
expect(masterGain.setTargetAtTime).toHaveBeenCalledTimes(mutedCallCount);
audio.setMuted(false);
expect(masterGain.setTargetAtTime).toHaveBeenLastCalledWith(
0.8,
1,
config.fadeInSeconds
);
});
it('stays silent without piano samples while preserving eraser noise', () => {
const audio = new GardenAudio(makeConfig());
const vibe = VIBE_PRESETS[0];

View file

@ -34,12 +34,14 @@ export class GardenAudio {
private hasStarted = false;
private isDestroyed = false;
private isMuted = false;
private masterVolume: number;
private isGestureActive = false;
private hasQueuedPianoLoad = false;
private lastEraserAt = Number.NEGATIVE_INFINITY;
private lastVibeStingerAt = Number.NEGATIVE_INFINITY;
public constructor(private readonly config: GardenAudioConfig) {
this.masterVolume = clamp01(config.masterVolume);
this.graph = new GardenAudioGraph(config);
this.piano = new PianoSampler(config, this.graph);
this.noise = new NoiseBurstPlayer(config, this.graph);
@ -81,7 +83,7 @@ export class GardenAudio {
.then(() => {
if (this.graph.context === context && !this.isDestroyed && !this.isMuted) {
this.graph.unlock();
this.graph.setMasterGain(this.config.masterVolume, startupRampSeconds);
this.graph.setMasterGain(this.masterVolume, startupRampSeconds);
}
})
.catch((error) => {
@ -95,7 +97,7 @@ export class GardenAudio {
this.hasStarted = true;
this.applyVibe(vibe);
this.pianoEngine.prime(context.currentTime);
this.graph.setMasterGain(this.config.masterVolume, startupRampSeconds);
this.graph.setMasterGain(this.masterVolume, startupRampSeconds);
if (!this.hasQueuedPianoLoad) {
this.hasQueuedPianoLoad = true;
@ -132,13 +134,24 @@ export class GardenAudio {
}
public setMuted(isMuted: boolean): void {
if (this.isMuted === isMuted) {
return;
}
this.isMuted = isMuted;
this.graph.setMasterGain(
isMuted ? this.config.muteGain : this.config.masterVolume,
isMuted ? this.config.muteGain : this.masterVolume,
isMuted ? this.config.muteRampSeconds : this.config.fadeInSeconds
);
}
public setMasterVolume(masterVolume: number): void {
this.masterVolume = clamp01(masterVolume);
if (!this.isMuted) {
this.graph.setMasterGain(this.masterVolume, this.config.updateRampSeconds);
}
}
public beginGesture(): void {
const context = this.graph.context;
if (!context) {
@ -174,7 +187,9 @@ export class GardenAudio {
this.pianoEngine.renderLookahead({
vibe: snapshot.vibe,
now: context.currentTime,
activity: snapshot.isErasing ? 0 : this.energy.getLevel(),
activity: snapshot.isErasing
? this.config.eraser.pianoActivity
: this.energy.getLevel(),
});
this.updateDelay(snapshot);
}
@ -267,7 +282,7 @@ export class GardenAudio {
durationSeconds: this.config.eraser.durationSeconds,
gain: this.config.eraser.noiseGain * distanceActivity,
filterHz,
pan: 0,
pan: this.config.eraser.pan,
});
}
}

View file

@ -49,6 +49,14 @@ const countNotesBetween = (
notes.filter((note) => note.startTime >= startSeconds && note.startTime < endSeconds)
.length;
const getNoteKey = (note: PianoNote): string =>
[
note.startTime.toFixed(3),
note.midi,
note.role ?? 'none',
note.pan.toFixed(3),
].join(':');
describe('GenerativePianoEngine', () => {
it('plays quiet background music even when the garden is idle', () => {
const { engine, notes } = makeEngine();
@ -187,4 +195,55 @@ describe('GenerativePianoEngine', () => {
expect(second.notes).toEqual(first.notes);
});
it('does not duplicate notes across overlapping lookahead windows', () => {
const { engine, notes } = makeEngine();
engine.renderLookahead({
vibe: VIBE_PRESETS[0],
now: 0,
activity: 0.72,
lookaheadSeconds: getBeatSeconds() * 2,
});
engine.renderLookahead({
vibe: VIBE_PRESETS[0],
now: getBeatSeconds() * 0.5,
activity: 0.72,
lookaheadSeconds: getBeatSeconds() * 2,
});
const noteKeys = notes.map(getNoteKey);
expect(new Set(noteKeys).size).toBe(noteKeys.length);
});
it('keeps generated notes inside the audio contract', () => {
const { engine, notes } = makeEngine();
VIBE_PRESETS.forEach((vibe) => {
engine.cue(0);
engine.renderLookahead({
vibe,
now: 0,
activity: 1,
lookaheadSeconds: getBeatSeconds() * getBeatsPerBar() * 4,
});
});
notes.forEach((note) => {
expect(Number.isFinite(note.startTime)).toBe(true);
expect(note.midi).toBeGreaterThanOrEqual(21);
expect(note.midi).toBeLessThanOrEqual(108);
expect(note.velocity).toBeGreaterThan(0);
expect(note.velocity).toBeLessThanOrEqual(0.4);
expect(note.pan).toBeGreaterThanOrEqual(-1);
expect(note.pan).toBeLessThanOrEqual(1);
expect(note.durationSeconds).toBeGreaterThan(0);
expect(note.lowpassHz ?? gardenAudioConfig.piano.lowpassHz).toBeGreaterThanOrEqual(
gardenAudioConfig.piano.lowpassMinHz
);
expect(note.lowpassHz ?? gardenAudioConfig.piano.lowpassHz).toBeLessThanOrEqual(
gardenAudioConfig.piano.lowpassMaxHz
);
});
});
});

File diff suppressed because it is too large Load diff

View file

@ -9,8 +9,9 @@ export class NoiseBurstPlayer {
) {}
public play({ startTime, durationSeconds, gain, filterHz, pan }: NoiseBurst): void {
const { context, eventBus, noiseBuffer } = this.graph;
if (!context || !eventBus || !noiseBuffer) {
const { context, eventBus, noiseBus, noiseBuffer } = this.graph;
const outputBus = noiseBus ?? eventBus;
if (!context || !outputBus || !noiseBuffer) {
return;
}
@ -25,7 +26,7 @@ export class NoiseBurstPlayer {
const stopAt = scheduledStart + durationSeconds;
source.buffer = noiseBuffer;
filter.type = 'bandpass';
filter.type = this.config.noiseBurst.filterType;
filter.frequency.setValueAtTime(filterHz, scheduledStart);
filter.Q.value = this.config.noiseBurst.filterQ;
envelope.gain.setValueAtTime(this.config.noiseBurst.silentGain, scheduledStart);
@ -39,7 +40,7 @@ export class NoiseBurstPlayer {
source.connect(filter);
filter.connect(envelope);
envelope.connect(panner);
panner.connect(eventBus);
panner.connect(outputBus);
source.start(
scheduledStart,
Math.random() * this.config.noiseBurst.offsetRandomSeconds

View file

@ -68,6 +68,7 @@ const makeSampler = async (context: AudioContext): Promise<PianoSampler> => {
context,
delayInput: null,
eventBus,
getPianoBus: vi.fn(() => eventBus),
} as unknown as GardenAudioGraph;
return new PianoSampler(gardenAudioConfig, graph);

View file

@ -38,10 +38,12 @@ export class PianoSampler {
startTime,
durationSeconds,
pan,
role,
delaySend = 0,
lowpassHz = this.config.piano.lowpassHz,
}: PianoNote): void {
const { context, eventBus, delayInput } = this.graph;
const { context, delayInput } = this.graph;
const eventBus = this.graph.getPianoBus(role);
if (!context || !eventBus) {
return;
}
@ -88,7 +90,7 @@ export class PianoSampler {
Math.pow(2, (midi - sample.midi) / this.config.piano.pitchSemitonesPerOctave),
scheduledStart
);
filter.type = 'lowpass';
filter.type = this.config.piano.filterType;
filter.frequency.setValueAtTime(
clamp(lowpassHz, this.config.piano.lowpassMinHz, this.config.piano.lowpassMaxHz),
scheduledStart

View file

@ -1,3 +1,4 @@
import { gardenAudioConfig } from './garden-audio-config';
import type { LoadedPianoSample } from './garden-audio-types';
interface PianoSampleDefinition {
@ -11,8 +12,6 @@ export interface PianoSampleLoadProgress {
sample?: PianoSampleDefinition;
}
const sampleBaseUrl = `${import.meta.env.BASE_URL}audio/piano/`;
const sampleFiles: Array<[fileName: string, midi: number]> = [
['A0v12.m4a', 21],
['C1v12.m4a', 24],
@ -49,7 +48,7 @@ const sampleFiles: Array<[fileName: string, midi: number]> = [
const pianoSampleDefinitions: Array<PianoSampleDefinition> = sampleFiles
.map(([fileName, midi]) => ({
midi,
url: `${sampleBaseUrl}${fileName}`,
url: `${gardenAudioConfig.piano.sampleBaseUrl}${fileName}`,
}))
.sort((a, b) => a.midi - b.midi);
@ -68,7 +67,11 @@ export const preloadPianoSamples = (
);
}
const decodeContext = new OfflineAudioContextConstructor(1, 1, 44_100);
const decodeContext = new OfflineAudioContextConstructor(
gardenAudioConfig.piano.preloadDecode.channels,
gardenAudioConfig.piano.preloadDecode.frames,
gardenAudioConfig.piano.preloadDecode.sampleRateHz
);
return loadPianoSamples(decodeContext, onProgress);
};