andras/fleeting-garden
andras/fleeting-garden
Watch
1
0
Fork
You've already forked fleeting-garden
 0
Code Issues Pull requests 10 Projects Releases Packages Wiki Activity Actions

clean up

Browse source
This commit is contained in:
Andras Schmelczer 2026-05-18 08:11:52 +01:00
commit 15e99380b5
84 changed files with 1659 additions and 4726 deletions
Download patch file Download diff file Expand all files Collapse all files

252
src/audio/garden-audio-config.ts
View file

@ -8,201 +8,6 @@ export interface GardenAudioChord {
quality: GardenAudioChordQuality;
}
interface GardenAudioStyleVoice {
scaleDegreeOffset: number;
velocityMultiplier: number;
panOffset: number;
}
export interface GardenAudioRegister {
midiMin: number;
midiMax: number;
preferredMidi: number;
pan: number;
}
export interface GardenAudioStylePool extends GardenAudioRegister {
scaleDegrees: Array<number>;
}
interface GardenAudioGenerativePianoConfig {
stylePools: [GardenAudioStylePool, GardenAudioStylePool, GardenAudioStylePool];
padRegisters: [GardenAudioRegister, GardenAudioRegister, GardenAudioRegister];
chordVoicings: {
majorOpen: Array<number>;
minorOpen: Array<number>;
majorClosed: Array<number>;
minorClosed: Array<number>;
};
vibeChangeStinger: {
velocities: [number, number, number];
pans: [number, number, number];
delaySends: [number, number, number];
lowpassExpression: number;
};
highActivityExtra: {
barOffset: number;
expressionMultiplier: number;
};
padChord: {
velocities: [number, number, number];
expressionVelocityWeight: number;
delaySend: number;
lowpassExpressionWeight: number;
};
supportNote: {
velocityBase: number;
velocityExpressionWeight: number;
durationBaseSeconds: number;
durationExpressionSeconds: number;
delaySendBase: number;
delaySendExpressionWeight: number;
lowpassExpressionWeight: number;
expressionThreshold: number;
offsetsByStyle: [Array<number>, Array<number>, Array<number>];
};
textureNote: {
velocityBase: number;
velocityExpressionWeight: number;
durationBaseSeconds: number;
durationExpressionSeconds: number;
delaySendBase: number;
delaySendExpressionWeight: number;
idleExpressionThreshold: number;
mediumExpressionThreshold: number;
intenseSpacing: number;
idlePhase: number;
};
gestureAccent: {
rotationStrengthMultiplier: number;
quantizeStepLookahead: number;
velocityBase: number;
velocityStrengthWeight: number;
durationBaseSeconds: number;
durationStrengthSeconds: number;
delaySend: number;
};
touchNote: {
registerBiasManiaAmount: number;
velocityBase: number;
velocityStrengthWeight: number;
durationBaseSeconds: number;
durationStrengthSeconds: number;
delaySend: number;
lowpassBaseExpression: number;
lowpassStrengthWeight: number;
};
brushPhrase: {
initialMotifOffset: number;
energyDecaySeconds: number;
maniaDecaySeconds: number;
fadeMinimumLifetimeSeconds: number;
layerIntensityBase: number;
layerIntensityManiaWeight: number;
frameActivityWeight: number;
frameManiaWeight: number;
};
brushStream: {
inferredManiaThreshold: number;
inferredManiaRange: number;
registerManiaShift: number;
chordToneEverySteps: number;
durationBaseSeconds: number;
durationIntensitySeconds: number;
durationManiaSeconds: number;
durationMinSeconds: number;
durationMaxSeconds: number;
delaySendBase: number;
delaySendIntensityWeight: number;
delaySendManiaWeight: number;
delaySendMin: number;
delaySendMax: number;
velocityBase: number;
velocityIntensityWeight: number;
lowpassBaseExpression: number;
lowpassIntensityWeight: number;
lowpassManiaWeight: number;
manicThreshold: number;
intenseThreshold: number;
activeThreshold: number;
};
brushStreamEcho: {
maniaThreshold: number;
stepModulo: number;
stepRemainder: number;
intensityThreshold: number;
octaveSemitones: number;
maxMidi: number;
velocityBase: number;
velocityIntensityWeight: number;
durationMinSeconds: number;
durationScale: number;
panScale: number;
delaySendMin: number;
delaySendScale: number;
lowpassBaseExpression: number;
lowpassManiaWeight: number;
};
brushMotif: {
highThreshold: number;
mediumThreshold: number;
highOffset: number;
mediumOffset: number;
lowOffset: number;
minOffset: number;
maxOffset: number;
};
registerBias: {
maniaShiftSemitones: number;
midiMin: number;
midiMaxForMin: number;
minimumSpan: number;
midiMax: number;
};
candidateOctaveSearch: {
min: number;
max: number;
};
stylePanOffsetScale: number;
lowpass: {
midiBase: number;
midiRange: number;
midiLiftHz: number;
expressionBase: number;
expressionWeight: number;
};
styleRotationBars: number;
chordBars: number;
supportBarSpacing: number;
supportBarOffset: number;
idleTextureBarSpacing: number;
mediumTextureBarSpacing: number;
textureBeat: number;
highActivityExtraBeat: number;
highActivityExtraThreshold: number;
noteScorePreferenceWeight: number;
noteScoreRegisterWeight: number;
noteScoreChordToneWeight: number;
noteScoreRepeatPenalty: number;
gestureAccentMinIntervalSeconds: number;
strokeAccentMinSteps: number;
strokeAccentThreshold: number;
stingerDurationSeconds: number;
stingerSpacingSeconds: number;
maxBrushPhraseLayers: number;
maxBrushStreamNotesPerBar: number;
brushLayerBaseSeconds: number;
brushLayerEnergySeconds: number;
brushLayerMinIntensity: number;
brushStreamIdleIntervalBeats: number;
brushStreamActiveIntervalBeats: number;
brushStreamIntenseIntervalBeats: number;
brushStreamManicIntervalBeats: number;
brushMotifMaxSteps: number;
brushMotifCanonDelaySeconds: number;
padDurationBarScale: number;
}
export interface GardenAudioVibeProfile {
rootMidi: number;
scale: Array<number>;
@ -215,7 +20,6 @@ export interface GardenAudioConfig {
masterVolume: number;
fadeInSeconds: number;
updateRampSeconds: number;
highPassFrequencyHz: number;
delay: {
timeSeconds: number;
feedback: number;
@ -228,44 +32,24 @@ export interface GardenAudioConfig {
outputBase: number;
outputActivityDuck: number;
timeRampSeconds: number;
feedbackHighPassHz: number;
feedbackLowPassHz: number;
returnLowPassHz: number;
};
piano: {
maxVoices: number;
filterType: BiquadFilterType;
gain: number;
sustainSeconds: number;
sustainLevel: number;
releaseSeconds: number;
lowpassHz: number;
filterQ: number;
gainAttackSeconds: number;
lowpassMaxHz: number;
lowpassMinHz: number;
minDurationSeconds: number;
minFadeSeconds: number;
minGain: number;
pitchSemitonesPerOctave: number;
scheduleAheadSeconds: number;
sustainBase: number;
sustainVelocityRange: number;
tailStopExtraSeconds: number;
voiceStealFadeSeconds: number;
voiceStealStopSeconds: number;
sampleBaseUrl: string;
preloadDecode: {
channels: number;
frames: number;
sampleRateHz: number;
};
};
rhythm: {
bpm: number;
stepsPerBeat: number;
stepsPerBar: number;
lookaheadSeconds: number;
sparseActivity: number;
};
eraser: {
@ -285,32 +69,11 @@ export interface GardenAudioConfig {
strokeDecaySeconds: number;
};
graph: {
closeGain: number;
closeRampSeconds: number;
delayMaxSeconds: number;
eventBusGain: number;
noiseMax: number;
noiseMin: number;
unlockTickFrequencyHz: number;
unlockTickSeconds: number;
unlockTickType: OscillatorType;
latencyHint: AudioContextLatencyCategory;
outputFilterType: BiquadFilterType;
noiseBufferChannels: number;
noiseBufferDurationSeconds: number;
pianoBusGains: Record<PianoNoteRole, number>;
pianoBusActivityDucking: Record<PianoNoteRole, number>;
noiseBusGain: number;
compressor: {
thresholdDb: number;
kneeDb: number;
ratio: number;
attackSeconds: number;
releaseSeconds: number;
};
};
input: {
fallbackFrameSeconds: number;
fullActivitySpeed: number;
activityNoiseFloorSpeed: number;
activityCurve: number;
@ -321,22 +84,7 @@ export interface GardenAudioConfig {
manicActivityThreshold: number;
manicReleaseThreshold: number;
maniaSmoothingSeconds: number;
minElapsedSeconds: number;
};
muteGain: number;
muteRampSeconds: number;
noiseBurst: {
attackSeconds: number;
filterQ: number;
offsetRandomSeconds: number;
scheduleAheadSeconds: number;
silentGain: number;
filterType: BiquadFilterType;
};
startDelaySeconds: number;
vibeChangeStingerMinIntervalSeconds: number;
generativePiano: GardenAudioGenerativePianoConfig;
styleVoices: [GardenAudioStyleVoice, GardenAudioStyleVoice, GardenAudioStyleVoice];
}
export const gardenAudioConfig: GardenAudioConfig = appConfig.audio;

49
src/audio/garden-audio-energy.test.ts
View file

@ -1,49 +0,0 @@
import { describe, expect, it } from 'vitest';
import { gardenAudioConfig } from './garden-audio-config';
import { GardenAudioEnergy } from './garden-audio-energy';
describe('GardenAudioEnergy', () => {
it('suspends activity but keeps a fading level when the gesture ends', () => {
const energy = new GardenAudioEnergy(gardenAudioConfig);
energy.beginGesture(0);
energy.recordStroke(0.8, 0.1);
energy.update(0.1);
energy.update(0.2);
const levelBeforeLift = energy.getLevel();
expect(energy.getActivity()).toBeGreaterThan(0);
energy.endGesture();
expect(energy.getActivity()).toBe(0);
expect(energy.getLevel()).toBe(levelBeforeLift);
energy.update(0.3);
expect(energy.getLevel()).toBeLessThan(levelBeforeLift);
expect(energy.getLevel()).toBeGreaterThan(0);
});
it('uses recent stroke intensity rather than gesture duration alone', () => {
const energy = new GardenAudioEnergy(gardenAudioConfig);
energy.beginGesture(0);
energy.recordStroke(1, 0.1);
energy.update(0.1);
energy.update(0.2);
const activeLevel = energy.getActivity();
energy.update(1.2);
expect(energy.getActivity()).toBeLessThan(activeLevel);
});
it('raises activity immediately when a stroke is recorded', () => {
const energy = new GardenAudioEnergy(gardenAudioConfig);
energy.beginGesture(0);
energy.recordStroke(0.12, 0.05);
expect(energy.getActivity()).toBeGreaterThan(0.09);
});
});

5
src/audio/garden-audio-energy.ts
View file

@ -1,4 +1,4 @@
import { clamp01 } from '../utils/clamp';
import { approach, clamp01 } from '../utils/math';
import type { GardenAudioConfig } from './garden-audio-config';
export class GardenAudioEnergy {
@ -59,8 +59,7 @@ export class GardenAudioEnergy {
} else if (target > this.energy) {
timeConstant = this.config.energy.attackSeconds;
}
const amount = 1 - Math.exp(-elapsedSeconds / timeConstant);
this.energy += (target - this.energy) * amount;
this.energy = approach(this.energy, target, elapsedSeconds, timeConstant);
}
public getActivity(): number {

95
src/audio/garden-audio-gesture-state.test.ts
View file

@ -1,95 +0,0 @@
import { describe, expect, it } from 'vitest';
import { gardenAudioConfig } from './garden-audio-config';
import { GardenAudioGestureState } from './garden-audio-gesture-state';
import type { GardenAudioStrokeMetrics } from './garden-audio-input';
const makeMetrics = ({
elapsedSeconds,
normalizedDistance,
}: {
elapsedSeconds: number;
normalizedDistance: number;
}): GardenAudioStrokeMetrics => ({
distancePixels: normalizedDistance * 1000,
elapsedSeconds,
normalizedDistance,
normalizedSpeed: normalizedDistance / elapsedSeconds,
});
describe('GardenAudioGestureState', () => {
it('ignores tiny jitter below the activity speed floor', () => {
const state = new GardenAudioGestureState(gardenAudioConfig.input);
state.beginGesture();
expect(
state.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.1,
normalizedDistance: 0.001,
}),
}).activity
).toBe(0);
});
it('normalizes equal drawing speeds across pointer sample rates', () => {
const lowRate = new GardenAudioGestureState(gardenAudioConfig.input);
const highRate = new GardenAudioGestureState(gardenAudioConfig.input);
lowRate.beginGesture();
highRate.beginGesture();
const lowRateFrame = lowRate.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.1,
normalizedDistance: 0.07,
}),
});
let highRateFrame = { activity: 0, maniaAmount: 0 };
for (let index = 0; index < 5; index += 1) {
highRateFrame = highRate.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.02,
normalizedDistance: 0.014,
}),
});
}
expect(highRateFrame.activity).toBeCloseTo(lowRateFrame.activity, 5);
});
it('holds mania with hysteresis before releasing', () => {
const state = new GardenAudioGestureState(gardenAudioConfig.input);
state.beginGesture();
const manicFrame = state.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.25,
normalizedDistance: 0.3,
}),
});
expect(manicFrame.maniaAmount).toBeGreaterThan(0);
const heldFrame = state.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.06,
normalizedDistance: 0.04,
}),
});
expect(heldFrame.maniaAmount).toBeGreaterThan(0);
const releasedFrame = state.recordStroke({
metrics: makeMetrics({
elapsedSeconds: 0.7,
normalizedDistance: 0,
}),
});
expect(releasedFrame.maniaAmount).toBeLessThan(heldFrame.maniaAmount);
});
});

17
src/audio/garden-audio-gesture-state.ts
View file

@ -1,4 +1,4 @@
import { clamp, clamp01 } from '../utils/clamp';
import { approach, clamp, clamp01, smoothstep } from '../utils/math';
import type { GardenAudioConfig } from './garden-audio-config';
import type { GardenAudioStrokeMetrics } from './garden-audio-input';
@ -92,18 +92,3 @@ export class GardenAudioGestureState {
return clamp(activity * distanceAmount, 0, this.inputConfig.activitySoftCeiling);
}
}
const approach = (
current: number,
target: number,
elapsedSeconds: number,
timeConstantSeconds: number
): number => {
const amount = 1 - Math.exp(-elapsedSeconds / Math.max(0.001, timeConstantSeconds));
return current + (target - current) * amount;
};
const smoothstep = (edge0: number, edge1: number, value: number): number => {
const amount = clamp01((value - edge0) / (edge1 - edge0));
return amount * amount * (3 - 2 * amount);
};

92
src/audio/garden-audio-graph.test.ts
View file

@ -1,92 +0,0 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { VIBE_PRESETS } from '../vibes';
import { gardenAudioConfig } from './garden-audio-config';
import { GardenAudioGraph } from './garden-audio-graph';
class FakeAudioParam {
public value = 0;
public setTargetAtTime = vi.fn();
}
class FakeAudioNode {
public readonly gain = new FakeAudioParam();
public readonly frequency = new FakeAudioParam();
public readonly threshold = new FakeAudioParam();
public readonly knee = new FakeAudioParam();
public readonly ratio = new FakeAudioParam();
public readonly attack = new FakeAudioParam();
public readonly release = new FakeAudioParam();
public readonly delayTime = new FakeAudioParam();
public readonly connections: Array<unknown> = [];
public type = '';
public connect(target: unknown): unknown {
this.connections.push(target);
return target;
}
}
class FakeAudioBuffer {
private readonly data: Float32Array;
public constructor(length: number) {
this.data = new Float32Array(length);
}
public getChannelData(): Float32Array {
return this.data;
}
}
class FakeAudioContext {
public readonly currentTime = 1;
public readonly destination = new FakeAudioNode() as unknown as AudioDestinationNode;
public readonly sampleRate = 16;
public readonly state = 'running';
public readonly compressors: Array<FakeAudioNode> = [];
public createGain(): GainNode {
return new FakeAudioNode() as unknown as GainNode;
}
public createBiquadFilter(): BiquadFilterNode {
return new FakeAudioNode() as unknown as BiquadFilterNode;
}
public createDelay(): DelayNode {
return new FakeAudioNode() as unknown as DelayNode;
}
public createDynamicsCompressor(): DynamicsCompressorNode {
const node = new FakeAudioNode();
this.compressors.push(node);
return node as unknown as DynamicsCompressorNode;
}
public createBuffer(_channels: number, length: number): AudioBuffer {
return new FakeAudioBuffer(length) as unknown as AudioBuffer;
}
}
describe('GardenAudioGraph', () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it('builds controlled output, role buses, and delay automation', () => {
vi.stubGlobal('AudioContext', FakeAudioContext);
const graph = new GardenAudioGraph(gardenAudioConfig);
const context = graph.ensureContext(true) as unknown as FakeAudioContext;
expect(context.compressors).toHaveLength(1);
expect(graph.getPianoBus('pad')).not.toBeNull();
expect(graph.getPianoBus('pad')).not.toBe(graph.getPianoBus('gesture'));
expect(graph.noiseBus).not.toBeNull();
graph.updateDelay(VIBE_PRESETS[0].audio, 1);
expect(graph.getPianoBus('pad')?.gain.setTargetAtTime).toHaveBeenCalled();
expect(graph.getPianoBus('brush')?.gain.setTargetAtTime).toHaveBeenCalled();
});
});

137
src/audio/garden-audio-graph.ts
View file

@ -1,7 +1,38 @@
import { clamp } from '../utils/clamp';
import { GardenAudioConfig, GardenAudioVibeProfile } from './garden-audio-config';
import { clamp } from '../utils/math';
import { isIosLike } from './audio-platform';
import type { GardenAudioConfig, GardenAudioVibeProfile } from './garden-audio-config';
import type { PianoNoteRole } from './garden-audio-types';
const outputHighPassFrequencyHz = 45;
const graphTuning = {
closeGain: 0.0001,
closeRampSeconds: 0.015,
delayMaxSeconds: 2,
eventBusGain: 1,
noiseMax: 1,
noiseMin: -1,
latencyHint: 'interactive' as AudioContextLatencyCategory,
outputFilterType: 'highpass' as BiquadFilterType,
noiseBufferChannels: 1,
noiseBufferDurationSeconds: 1,
unlockTickFrequencyHz: 440,
unlockTickGain: 0.0001,
unlockTickSeconds: 0.025,
unlockTickType: 'sine' as OscillatorType,
compressor: {
thresholdDb: -18,
kneeDb: 18,
ratio: 2.1,
attackSeconds: 0.018,
releaseSeconds: 0.18,
},
};
const delayFilterTuning = {
feedbackHighPassHz: 180,
feedbackLowPassHz: 5200,
returnLowPassHz: 6200,
};
export class GardenAudioGraph {
public context: AudioContext | null = null;
public eventBus: GainNode | null = null;
@ -13,6 +44,8 @@ export class GardenAudioGraph {
private delayNode: DelayNode | null = null;
private delayFeedback: GainNode | null = null;
private delayOutput: GainNode | null = null;
private mediaStreamDestination: MediaStreamAudioDestinationNode | null = null;
private mediaStreamElement: HTMLAudioElement | null = null;
private readonly pianoBuses = new Map<PianoNoteRole, GainNode>();
public constructor(private readonly config: GardenAudioConfig) {}
@ -34,7 +67,7 @@ export class GardenAudioGraph {
let context: AudioContext;
try {
context = new AudioContextConstructor({
latencyHint: this.config.graph.latencyHint,
latencyHint: graphTuning.latencyHint,
});
} catch {
context = new AudioContextConstructor();
@ -42,22 +75,30 @@ export class GardenAudioGraph {
const masterGain = context.createGain();
const highPass = context.createBiquadFilter();
const compressor = context.createDynamicsCompressor();
const mediaStreamDestination = isIosLike()
? context.createMediaStreamDestination()
: null;
masterGain.gain.value = 0;
highPass.type = this.config.graph.outputFilterType;
highPass.frequency.value = this.config.highPassFrequencyHz;
compressor.threshold.value = this.config.graph.compressor.thresholdDb;
compressor.knee.value = this.config.graph.compressor.kneeDb;
compressor.ratio.value = this.config.graph.compressor.ratio;
compressor.attack.value = this.config.graph.compressor.attackSeconds;
compressor.release.value = this.config.graph.compressor.releaseSeconds;
highPass.type = graphTuning.outputFilterType;
highPass.frequency.value = outputHighPassFrequencyHz;
compressor.threshold.value = graphTuning.compressor.thresholdDb;
compressor.knee.value = graphTuning.compressor.kneeDb;
compressor.ratio.value = graphTuning.compressor.ratio;
compressor.attack.value = graphTuning.compressor.attackSeconds;
compressor.release.value = graphTuning.compressor.releaseSeconds;
masterGain.connect(highPass);
highPass.connect(compressor);
compressor.connect(context.destination);
if (mediaStreamDestination) {
compressor.connect(mediaStreamDestination);
} else {
compressor.connect(context.destination);
}
this.context = context;
this.masterGain = masterGain;
this.mediaStreamDestination = mediaStreamDestination;
this.noiseBuffer = this.createNoiseBuffer(context);
this.createDelay(context, masterGain);
this.createBuses(context, masterGain);
@ -77,17 +118,13 @@ export class GardenAudioGraph {
const source = this.context.createOscillator();
const gain = this.context.createGain();
source.type = this.config.graph.unlockTickType;
source.frequency.setValueAtTime(this.config.graph.unlockTickFrequencyHz, now);
gain.gain.setValueAtTime(this.config.piano.minGain, now);
gain.gain.exponentialRampToValueAtTime(
this.config.piano.minGain,
now + this.config.graph.unlockTickSeconds
);
source.type = graphTuning.unlockTickType;
source.frequency.setValueAtTime(graphTuning.unlockTickFrequencyHz, now);
gain.gain.setValueAtTime(graphTuning.unlockTickGain, now);
source.connect(gain);
gain.connect(this.context.destination);
source.start(now);
source.stop(now + this.config.graph.unlockTickSeconds);
source.stop(now + graphTuning.unlockTickSeconds);
source.addEventListener(
'ended',
() => {
@ -110,6 +147,38 @@ export class GardenAudioGraph {
);
}
public startMediaElementOutput(): void {
if (!this.mediaStreamDestination) {
return;
}
const mediaElement = this.ensureMediaStreamElement();
const playPromise = mediaElement.play();
void playPromise?.catch(() => undefined);
}
private ensureMediaStreamElement(): HTMLAudioElement {
if (this.mediaStreamElement) {
return this.mediaStreamElement;
}
const mediaElement = document.createElement('audio');
mediaElement.autoplay = true;
mediaElement.volume = 1;
mediaElement.setAttribute('playsinline', '');
mediaElement.setAttribute('aria-hidden', 'true');
mediaElement.style.position = 'fixed';
mediaElement.style.width = '1px';
mediaElement.style.height = '1px';
mediaElement.style.opacity = '0';
mediaElement.style.pointerEvents = 'none';
mediaElement.style.left = '-9999px';
mediaElement.srcObject = this.mediaStreamDestination?.stream ?? null;
document.body.append(mediaElement);
this.mediaStreamElement = mediaElement;
return mediaElement;
}
public applyDelayProfile(profile: GardenAudioVibeProfile): void {
if (!this.context || !this.delayNode) {
return;
@ -167,9 +236,9 @@ export class GardenAudioGraph {
if (this.masterGain && context.state !== 'closed') {
this.masterGain.gain.setTargetAtTime(
this.config.graph.closeGain,
graphTuning.closeGain,
context.currentTime,
this.config.graph.closeRampSeconds
graphTuning.closeRampSeconds
);
}
@ -182,7 +251,7 @@ export class GardenAudioGraph {
private createDelay(context: AudioContext, masterGain: GainNode): void {
const delayInput = context.createGain();
const delayNode = context.createDelay(this.config.graph.delayMaxSeconds);
const delayNode = context.createDelay(graphTuning.delayMaxSeconds);
const delayFeedback = context.createGain();
const delayOutput = context.createGain();
const feedbackHighPass = context.createBiquadFilter();
@ -193,11 +262,11 @@ export class GardenAudioGraph {
delayFeedback.gain.value = this.config.delay.feedback;
delayOutput.gain.value = this.config.delay.wetGain;
feedbackHighPass.type = 'highpass';
feedbackHighPass.frequency.value = this.config.delay.feedbackHighPassHz;
feedbackHighPass.frequency.value = delayFilterTuning.feedbackHighPassHz;
feedbackLowPass.type = 'lowpass';
feedbackLowPass.frequency.value = this.config.delay.feedbackLowPassHz;
feedbackLowPass.frequency.value = delayFilterTuning.feedbackLowPassHz;
returnLowPass.type = 'lowpass';
returnLowPass.frequency.value = this.config.delay.returnLowPassHz;
returnLowPass.frequency.value = delayFilterTuning.returnLowPassHz;
delayInput.connect(delayNode);
delayNode.connect(feedbackHighPass);
@ -216,7 +285,7 @@ export class GardenAudioGraph {
private createBuses(context: AudioContext, masterGain: GainNode): void {
const eventBus = context.createGain();
eventBus.gain.value = this.config.graph.eventBusGain;
eventBus.gain.value = graphTuning.eventBusGain;
eventBus.connect(masterGain);
this.eventBus = eventBus;
this.pianoBuses.clear();
@ -249,12 +318,11 @@ export class GardenAudioGraph {
private createNoiseBuffer(context: AudioContext): AudioBuffer {
const buffer = context.createBuffer(
appPositiveInteger(this.config.graph.noiseBufferChannels),
appPositiveInteger(graphTuning.noiseBufferChannels),
Math.max(
1,
Math.floor(
context.sampleRate *
Math.max(0.001, this.config.graph.noiseBufferDurationSeconds)
context.sampleRate * Math.max(0.001, graphTuning.noiseBufferDurationSeconds)
)
),
context.sampleRate
@ -263,8 +331,8 @@ export class GardenAudioGraph {
for (let index = 0; index < data.length; index++) {
data[index] =
this.config.graph.noiseMin +
Math.random() * (this.config.graph.noiseMax - this.config.graph.noiseMin);
graphTuning.noiseMin +
Math.random() * (graphTuning.noiseMax - graphTuning.noiseMin);
}
return buffer;
@ -280,6 +348,13 @@ export class GardenAudioGraph {
this.delayNode = null;
this.delayFeedback = null;
this.delayOutput = null;
this.mediaStreamDestination = null;
if (this.mediaStreamElement) {
this.mediaStreamElement.pause();
this.mediaStreamElement.srcObject = null;
this.mediaStreamElement.remove();
this.mediaStreamElement = null;
}
this.pianoBuses.clear();
}
}

52
src/audio/garden-audio-input.test.ts
View file

@ -1,52 +0,0 @@
import { describe, expect, it } from 'vitest';
import { gardenAudioConfig } from './garden-audio-config';
import { getStrokeMetrics } from './garden-audio-input';
describe('getStrokeMetrics', () => {
it('normalizes stroke distance against canvas size', () => {
const standardDensity = getStrokeMetrics(
{
vibe: {} as never,
from: [0, 0],
to: [100, 0],
canvasSize: [1000, 500],
elapsedSeconds: 0.1,
isErasing: false,
},
gardenAudioConfig.input
);
const highDensity = getStrokeMetrics(
{
vibe: {} as never,
from: [0, 0],
to: [200, 0],
canvasSize: [2000, 1000],
elapsedSeconds: 0.1,
isErasing: false,
},
gardenAudioConfig.input
);
expect(highDensity.normalizedDistance).toBeCloseTo(
standardDensity.normalizedDistance
);
expect(highDensity.normalizedSpeed).toBeCloseTo(standardDensity.normalizedSpeed);
});
it('uses configured elapsed-time floors for missing or invalid samples', () => {
const metrics = getStrokeMetrics(
{
vibe: {} as never,
from: [0, 0],
to: [10, 0],
elapsedSeconds: 0,
isErasing: false,
},
gardenAudioConfig.input
);
expect(metrics.elapsedSeconds).toBe(gardenAudioConfig.input.fallbackFrameSeconds);
expect(Number.isFinite(metrics.normalizedSpeed)).toBe(true);
});
});

19
src/audio/garden-audio-input.ts
View file

@ -1,7 +1,8 @@
import type { GardenAudioConfig } from './garden-audio-config';
import type { GardenAudioStroke } from './garden-audio-types';
const fallbackNormalizationPixels = 1000;
const fallbackFrameSeconds = 1 / 60;
const minElapsedSeconds = 0.001;
export interface GardenAudioStrokeMetrics {
distancePixels: number;
@ -10,14 +11,11 @@ export interface GardenAudioStrokeMetrics {
normalizedSpeed: number;
}
export const getStrokeMetrics = (
stroke: GardenAudioStroke,
inputConfig: GardenAudioConfig['input']
): GardenAudioStrokeMetrics => {
export const getStrokeMetrics = (stroke: GardenAudioStroke): GardenAudioStrokeMetrics => {
const dx = stroke.to[0] - stroke.from[0];
const dy = stroke.to[1] - stroke.from[1];
const distancePixels = Math.hypot(dx, dy);
const elapsedSeconds = getElapsedSeconds(stroke, inputConfig);
const elapsedSeconds = getElapsedSeconds(stroke);
const normalizedDistance = distancePixels / getStrokeNormalizationPixels(stroke);
return {
@ -28,19 +26,16 @@ export const getStrokeMetrics = (
};
};
const getElapsedSeconds = (
stroke: GardenAudioStroke,
inputConfig: GardenAudioConfig['input']
): number => {
const getElapsedSeconds = (stroke: GardenAudioStroke): number => {
if (
stroke.elapsedSeconds !== undefined &&
Number.isFinite(stroke.elapsedSeconds) &&
stroke.elapsedSeconds > 0
) {
return Math.max(inputConfig.minElapsedSeconds, stroke.elapsedSeconds);
return Math.max(minElapsedSeconds, stroke.elapsedSeconds);
}
return inputConfig.fallbackFrameSeconds;
return fallbackFrameSeconds;
};
const getStrokeNormalizationPixels = (stroke: GardenAudioStroke): number => {

29
src/audio/garden-audio-music.ts
View file

@ -1,9 +1,14 @@
import { VibePreset } from '../vibes';
import {
GardenAudioChord,
gardenAudioConfig,
GardenAudioVibeProfile,
} from './garden-audio-config';
import type { VibePreset } from '../vibes';
import type { GardenAudioChord, GardenAudioVibeProfile } from './garden-audio-config';
export const PITCH_SEMITONES_PER_OCTAVE = 12;
const chordVoicings = {
majorOpen: [0, 7, 12, 16],
minorOpen: [0, 7, 12, 15],
majorClosed: [0, 4, 7, 12, 16],
minorClosed: [0, 3, 7, 12, 15],
};
export const getVibeProfile = (vibe: VibePreset): GardenAudioVibeProfile => vibe.audio;
@ -12,14 +17,12 @@ export const getChordIntervals = (
openVoicing: boolean
): Array<number> => {
if (openVoicing) {
return chord.quality === 'major'
? gardenAudioConfig.generativePiano.chordVoicings.majorOpen
: gardenAudioConfig.generativePiano.chordVoicings.minorOpen;
return chord.quality === 'major' ? chordVoicings.majorOpen : chordVoicings.minorOpen;
}
return chord.quality === 'major'
? gardenAudioConfig.generativePiano.chordVoicings.majorClosed
: gardenAudioConfig.generativePiano.chordVoicings.minorClosed;
? chordVoicings.majorClosed
: chordVoicings.minorClosed;
};
export const degreeToSemitone = (
@ -29,7 +32,5 @@ export const degreeToSemitone = (
const scaleIndex =
((degree % profile.scale.length) + profile.scale.length) % profile.scale.length;
const octave = Math.floor(degree / profile.scale.length);
return (
profile.scale[scaleIndex] + octave * gardenAudioConfig.piano.pitchSemitonesPerOctave
);
return profile.scale[scaleIndex] + octave * PITCH_SEMITONES_PER_OCTAVE;
};

326
src/audio/garden-audio.test.ts
View file

@ -1,326 +0,0 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { appConfig } from '../config';
import { VIBE_PRESETS } from '../vibes';
import { gardenAudioConfig, GardenAudioConfig } from './garden-audio-config';
type FakeScheduledSourceNode = {
start: ReturnType<typeof vi.fn>;
stop: ReturnType<typeof vi.fn>;
};
const calls = {
constructed: 0,
resumed: 0,
sourcesStarted: 0,
sources: [] as Array<FakeScheduledSourceNode>,
gains: [] as Array<FakeAudioNode>,
};
let contextState: AudioContextState = 'suspended';
let resumeError: Error | null = null;
let ErrorHandler: typeof import('../utils/error-handler').ErrorHandler;
let GardenAudio: typeof import('./garden-audio').GardenAudio;
let loadPianoSamples: typeof import('./piano-samples').loadPianoSamples;
let Severity: typeof import('../utils/error-handler').Severity;
class FakeAudioParam {
public value = 0;
public setTargetAtTime = vi.fn();
public setValueAtTime = vi.fn();
public exponentialRampToValueAtTime = vi.fn();
public cancelScheduledValues = vi.fn();
}
class FakeAudioNode {
public readonly gain = new FakeAudioParam();
public readonly frequency = new FakeAudioParam();
public readonly playbackRate = new FakeAudioParam();
public readonly Q = new FakeAudioParam();
public readonly threshold = new FakeAudioParam();
public readonly knee = new FakeAudioParam();
public readonly ratio = new FakeAudioParam();
public readonly attack = new FakeAudioParam();
public readonly release = new FakeAudioParam();
public readonly delayTime = new FakeAudioParam();
public readonly pan = new FakeAudioParam();
public type = '';
public addEventListener = vi.fn();
public connect = vi.fn();
public disconnect = vi.fn();
}
class FakeAudioBuffer {
private readonly data: Float32Array;
public constructor(length: number) {
this.data = new Float32Array(length);
}
public getChannelData(): Float32Array {
return this.data;
}
}
class FakeAudioContext {
public readonly currentTime = 1;
public readonly sampleRate = 16;
public readonly destination = new FakeAudioNode() as unknown as AudioDestinationNode;
public readonly decodeAudioData = vi.fn(async () => ({}) as AudioBuffer);
public constructor() {
calls.constructed += 1;
}
public get state(): AudioContextState {
return contextState;
}
public set state(state: AudioContextState) {
contextState = state;
}
public createGain(): GainNode {
const node = new FakeAudioNode();
calls.gains.push(node);
return node as unknown as GainNode;
}
public createBiquadFilter(): BiquadFilterNode {
return new FakeAudioNode() as unknown as BiquadFilterNode;
}
public createDelay(): DelayNode {
return new FakeAudioNode() as unknown as DelayNode;
}
public createDynamicsCompressor(): DynamicsCompressorNode {
return new FakeAudioNode() as unknown as DynamicsCompressorNode;
}
public createStereoPanner(): StereoPannerNode {
return new FakeAudioNode() as unknown as StereoPannerNode;
}
public createBuffer(_channels: number, length: number): AudioBuffer {
return new FakeAudioBuffer(length) as unknown as AudioBuffer;
}
public createBufferSource(): AudioBufferSourceNode {
const node = new FakeAudioNode() as unknown as AudioBufferSourceNode & {
buffer: AudioBuffer | null;
start: () => void;
stop: () => void;
};
node.buffer = null;
node.start = vi.fn(() => {
calls.sourcesStarted += 1;
}) as unknown as typeof node.start;
node.stop = vi.fn() as unknown as typeof node.stop;
calls.sources.push(node as unknown as FakeScheduledSourceNode);
return node;
}
public createOscillator(): OscillatorNode {
const node = new FakeAudioNode() as unknown as OscillatorNode & {
start: () => void;
stop: () => void;
};
node.start = vi.fn(() => {
calls.sourcesStarted += 1;
}) as unknown as typeof node.start;
node.stop = vi.fn() as unknown as typeof node.stop;
calls.sources.push(node as unknown as FakeScheduledSourceNode);
return node;
}
public async resume(): Promise<void> {
calls.resumed += 1;
if (resumeError) {
throw resumeError;
}
contextState = 'running';
}
}
const makeConfig = (): GardenAudioConfig => ({
...gardenAudioConfig,
});
describe('GardenAudio startup policy', () => {
beforeEach(async () => {
vi.resetModules();
({ ErrorHandler, Severity } = await import('../utils/error-handler'));
({ GardenAudio } = await import('./garden-audio'));
({ loadPianoSamples } = await import('./piano-samples'));
calls.constructed = 0;
calls.resumed = 0;
calls.sourcesStarted = 0;
calls.sources = [];
calls.gains = [];
contextState = 'suspended';
resumeError = null;
vi.stubGlobal('AudioContext', FakeAudioContext);
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('not loaded in tests')));
});
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
it('does not create an AudioContext from passive audio paths', () => {
const audio = new GardenAudio(makeConfig());
const vibe = VIBE_PRESETS[0];
audio.start(vibe);
audio.stroke({
vibe,
from: [0, 0],
to: [12, 0],
isErasing: false,
});
expect(calls.constructed).toBe(0);
});
it('only resumes a suspended context from a user gesture start', () => {
const audio = new GardenAudio(makeConfig());
const vibe = VIBE_PRESETS[0];
audio.start(vibe, { userGesture: true });
expect(calls.constructed).toBe(1);
expect(calls.resumed).toBe(1);
expect(contextState).toBe('running');
contextState = 'suspended';
audio.start(vibe);
audio.setMuted(false);
expect(calls.resumed).toBe(1);
});
it('reports AudioContext resume failures as warnings', async () => {
const audio = new GardenAudio(makeConfig());
const vibe = VIBE_PRESETS[0];
resumeError = new Error('resume rejected');
const addException = vi.spyOn(ErrorHandler, 'addException');
audio.start(vibe, { userGesture: true });
await Promise.resolve();
await Promise.resolve();
expect(addException).toHaveBeenCalledWith(resumeError, {
fallbackMessage: 'Could not resume audio playback.',
severity: Severity.WARNING,
});
});
it('updates live master gain from adjustable volume', () => {
const config = makeConfig();
const audio = new GardenAudio(config);
const vibe = VIBE_PRESETS[0];
audio.start(vibe, { userGesture: true });
const masterGain = calls.gains[0]?.gain;
if (!masterGain) {
throw new Error('Missing fake master gain');
}
audio.setMasterVolume(0.2);
expect(masterGain.setTargetAtTime).toHaveBeenLastCalledWith(
0.2,
1,
config.updateRampSeconds
);
audio.setMuted(true);
const mutedCallCount = masterGain.setTargetAtTime.mock.calls.length;
audio.setMasterVolume(0.8);
expect(masterGain.setTargetAtTime).toHaveBeenCalledTimes(mutedCallCount);
audio.setMuted(false);
expect(masterGain.setTargetAtTime).toHaveBeenLastCalledWith(
0.8,
1,
config.fadeInSeconds
);
});
it('stays silent without piano samples while preserving eraser noise', () => {
const audio = new GardenAudio(makeConfig());
const vibe = VIBE_PRESETS[0];
audio.start(vibe, { userGesture: true });
expect(calls.sourcesStarted).toBe(1);
audio.beginGesture();
audio.stroke({
vibe,
from: [30, 40],
to: [60, 60],
isErasing: false,
elapsedSeconds: 0.05,
});
expect(calls.sourcesStarted).toBe(1);
audio.stroke({
vibe,
from: [60, 60],
to: [75, 80],
isErasing: true,
elapsedSeconds: 0.05,
});
expect(calls.sourcesStarted).toBe(2);
});
it('quickly stops active piano voices when the vibe changes', async () => {
vi.stubGlobal(
'fetch',
vi.fn(async () => ({
arrayBuffer: async () => new ArrayBuffer(8),
ok: true,
}))
);
await loadPianoSamples(new FakeAudioContext() as unknown as AudioContext);
const audio = new GardenAudio(makeConfig());
const vibe = VIBE_PRESETS[0];
audio.start(vibe, { userGesture: true });
audio.beginGesture();
audio.stroke({
vibe,
from: [30, 40],
to: [90, 40],
elapsedSeconds: 0.05,
isErasing: false,
});
const activePianoSources = calls.sources.filter(
(source) => source.stop.mock.calls.length === 1
);
expect(activePianoSources.length).toBeGreaterThan(0);
const stopCounts = activePianoSources.map((source) => source.stop.mock.calls.length);
audio.changeVibe(VIBE_PRESETS[1], { userGesture: true });
const stoppedVoices = activePianoSources.filter(
(source, index) => source.stop.mock.calls.length === stopCounts[index] + 1
);
expect(stoppedVoices.length).toBeGreaterThan(0);
stoppedVoices.forEach((source) => {
expect(source.stop.mock.calls.at(-1)?.[0]).toBeCloseTo(
1 + appConfig.audio.piano.voiceStealStopSeconds,
3
);
});
});
});

116
src/audio/garden-audio.ts
View file

@ -1,7 +1,7 @@
import { clamp01 } from '../utils/clamp';
import { ErrorHandler, Severity } from '../utils/error-handler';
import { clamp01 } from '../utils/math';
import type { VibeId, VibePreset } from '../vibes';
import { GardenAudioConfig } from './garden-audio-config';
import type { GardenAudioConfig } from './garden-audio-config';
import { GardenAudioEnergy } from './garden-audio-energy';
import { GardenAudioGestureState } from './garden-audio-gesture-state';
import { GardenAudioGraph } from './garden-audio-graph';
@ -24,6 +24,12 @@ export type {
type AudioLifecycle = 'idle' | 'started' | 'destroyed';
const muteGain = 0.0001;
const muteRampSeconds = 0.02;
const brushUpPianoFinishSeconds = 1.2;
const brushUpPianoFadeSeconds = 1.1;
const vibeChangeStingerMinIntervalSeconds = 0.45;
export class GardenAudio {
private readonly graph: GardenAudioGraph;
private readonly piano: PianoSampler;
@ -36,7 +42,10 @@ export class GardenAudio {
private lifecycle: AudioLifecycle = 'idle';
private isMuted = false;
private isGestureActive = false;
private isPianoStoppedAfterGesture = false;
private fadePianoAfter: number | null = null;
private masterVolume: number;
private stopPianoAfter: number | null = null;
private lastEraserAt = Number.NEGATIVE_INFINITY;
private lastVibeStingerAt = Number.NEGATIVE_INFINITY;
@ -44,50 +53,55 @@ export class GardenAudio {
this.masterVolume = clamp01(config.masterVolume);
this.graph = new GardenAudioGraph(config);
this.piano = new PianoSampler(config, this.graph);
this.noise = new NoiseBurstPlayer(config, this.graph);
this.noise = new NoiseBurstPlayer(this.graph);
this.energy = new GardenAudioEnergy(config);
this.gestureState = new GardenAudioGestureState(config.input);
this.pianoEngine = new GenerativePianoEngine(config, (note) => this.piano.play(note));
}
public start(vibe: VibePreset, options: GardenAudioStartOptions = {}): void {
if (this.lifecycle === 'destroyed' || this.isMuted) {
const isUserGesture = options.userGesture === true;
if (this.lifecycle === 'destroyed') {
return;
}
const context = this.graph.ensureContext(options.userGesture === true);
const context = this.graph.ensureContext(isUserGesture);
if (!context) {
return;
}
const startupRampSeconds =
options.userGesture === true
? this.config.muteRampSeconds
: this.config.fadeInSeconds;
const startupRampSeconds = isUserGesture
? muteRampSeconds
: this.config.fadeInSeconds;
const needsResume = context.state !== 'running' && context.state !== 'closed';
let resumePromise: Promise<void> | null = null;
if (isUserGesture) {
this.graph.startMediaElementOutput();
this.graph.unlock();
}
if (needsResume) {
if (options.userGesture !== true) {
if (!isUserGesture) {
return;
}
resumePromise = context.resume();
}
if (options.userGesture === true) {
if (isUserGesture) {
this.graph.unlock();
}
if (resumePromise) {
void resumePromise
.then(() => {
if (
this.graph.context === context &&
this.lifecycle !== 'destroyed' &&
!this.isMuted
) {
if (this.graph.context === context && this.lifecycle !== 'destroyed') {
this.graph.unlock();
this.graph.setMasterGain(this.masterVolume, startupRampSeconds);
this.completeStart(vibe, {
context,
startupRampSeconds,
});
}
})
.catch((error) => {
@ -96,6 +110,32 @@ export class GardenAudio {
severity: Severity.WARNING,
});
});
return;
}
this.completeStart(vibe, {
context,
startupRampSeconds,
});
}
private completeStart(
vibe: VibePreset,
{
context,
startupRampSeconds,
}: {
context: AudioContext;
startupRampSeconds: number;
}
): void {
if (this.graph.context !== context || this.lifecycle === 'destroyed') {
return;
}
if (this.isMuted) {
this.graph.setMasterGain(muteGain, muteRampSeconds);
return;
}
this.lifecycle = 'started';
@ -111,7 +151,12 @@ export class GardenAudio {
this.pianoEngine.cue(context.currentTime);
}
})
.catch(() => undefined);
.catch((error) => {
ErrorHandler.addException(error, {
fallbackMessage: 'Could not load piano samples. Using synthesized audio.',
severity: Severity.WARNING,
});
});
}
}
@ -143,8 +188,8 @@ export class GardenAudio {
this.isMuted = isMuted;
this.graph.setMasterGain(
isMuted ? this.config.muteGain : this.masterVolume,
isMuted ? this.config.muteRampSeconds : this.config.fadeInSeconds
isMuted ? muteGain : this.masterVolume,
isMuted ? muteRampSeconds : this.config.fadeInSeconds
);
}
@ -162,6 +207,9 @@ export class GardenAudio {
}
this.isGestureActive = true;
this.isPianoStoppedAfterGesture = false;
this.fadePianoAfter = null;
this.stopPianoAfter = null;
this.gestureState.beginGesture();
this.energy.beginGesture(context.currentTime);
this.pianoEngine.beginGesture();
@ -170,6 +218,12 @@ export class GardenAudio {
public endGesture(): void {
this.gestureState.endGesture();
this.isGestureActive = false;
const context = this.graph.context;
this.isPianoStoppedAfterGesture = true;
this.fadePianoAfter = context
? context.currentTime + brushUpPianoFinishSeconds
: null;
this.stopPianoAfter = null;
this.energy.endGesture();
this.pianoEngine.endGesture();
}
@ -187,6 +241,21 @@ export class GardenAudio {
this.energy.silence();
}
if (!this.isGestureActive && this.isPianoStoppedAfterGesture) {
if (this.fadePianoAfter !== null && context.currentTime >= this.fadePianoAfter) {
this.piano.fadeAll(brushUpPianoFadeSeconds);
this.fadePianoAfter = null;
this.stopPianoAfter = context.currentTime + brushUpPianoFadeSeconds;
}
if (this.stopPianoAfter !== null && context.currentTime >= this.stopPianoAfter) {
this.piano.stopAll();
this.pianoEngine.reset();
this.stopPianoAfter = null;
}
this.updateDelay(snapshot);
return;
}
this.pianoEngine.renderLookahead({
vibe: snapshot.vibe,
now: context.currentTime,
@ -211,7 +280,7 @@ export class GardenAudio {
return;
}
const metrics = getStrokeMetrics(stroke, this.config.input);
const metrics = getStrokeMetrics(stroke);
const now = context.currentTime;
const frame = this.gestureState.recordStroke({ metrics });
@ -242,6 +311,9 @@ export class GardenAudio {
this.pianoEngine.reset();
this.currentVibeId = null;
this.isGestureActive = false;
this.isPianoStoppedAfterGesture = false;
this.fadePianoAfter = null;
this.stopPianoAfter = null;
this.lastEraserAt = Number.NEGATIVE_INFINITY;
this.lastVibeStingerAt = Number.NEGATIVE_INFINITY;
}
@ -253,7 +325,7 @@ export class GardenAudio {
}
const now = context.currentTime;
if (now - this.lastVibeStingerAt < this.config.vibeChangeStingerMinIntervalSeconds) {
if (now - this.lastVibeStingerAt < vibeChangeStingerMinIntervalSeconds) {
return;
}

249
src/audio/generative-piano.test.ts
View file

@ -1,249 +0,0 @@
import { describe, expect, it } from 'vitest';
import { VIBE_PRESETS } from '../vibes';
import { gardenAudioConfig } from './garden-audio-config';
import { PianoNote } from './garden-audio-types';
import { GenerativePianoEngine } from './generative-piano';
const makeEngine = () => {
const notes: Array<PianoNote> = [];
const engine = new GenerativePianoEngine(gardenAudioConfig, (note) => {
notes.push(note);
});
return { engine, notes };
};
const getBeatSeconds = (): number => 60 / gardenAudioConfig.rhythm.bpm;
const getBeatsPerBar = (): number =>
Math.round(
gardenAudioConfig.rhythm.stepsPerBar / gardenAudioConfig.rhythm.stepsPerBeat
);
const renderBars = (
engine: GenerativePianoEngine,
activity: number,
bars = 8,
now = 0
) => {
engine.renderLookahead({
vibe: VIBE_PRESETS[0],
now,
activity,
lookaheadSeconds: getBeatSeconds() * getBeatsPerBar() * bars,
});
};
const average = (values: Array<number>): number =>
values.reduce((sum, value) => sum + value, 0) / values.length;
const uniqueStartTimes = (notes: Array<PianoNote>): Array<string> =>
Array.from(new Set(notes.map((note) => note.startTime.toFixed(3))));
const countNotesBetween = (
notes: Array<PianoNote>,
startSeconds: number,
endSeconds: number
): number =>
notes.filter((note) => note.startTime >= startSeconds && note.startTime < endSeconds)
.length;
const getNoteKey = (note: PianoNote): string =>
[
note.startTime.toFixed(3),
note.midi,
note.role ?? 'none',
note.pan.toFixed(3),
].join(':');
describe('GenerativePianoEngine', () => {
it('plays quiet background music even when the garden is idle', () => {
const { engine, notes } = makeEngine();
renderBars(engine, 0);
expect(notes.length).toBeGreaterThan(0);
expect(notes.some((note) => note.durationSeconds > getBeatSeconds() * 6)).toBe(true);
expect(Math.max(...notes.map((note) => note.velocity))).toBeLessThan(0.12);
});
it('keeps the background sparse instead of filling every beat', () => {
const { engine, notes } = makeEngine();
renderBars(engine, 0, 4);
expect(uniqueStartTimes(notes).length).toBeLessThan(8);
});
it('lets activity add density without changing the beat grid', () => {
const idle = makeEngine();
const active = makeEngine();
const startDelaySeconds = 0.02;
renderBars(idle.engine, 0, 8);
renderBars(active.engine, 1, 8);
expect(active.notes.length).toBeGreaterThan(idle.notes.length);
active.notes.forEach((note) => {
const beatsFromStart = (note.startTime - startDelaySeconds) / getBeatSeconds();
expect(Math.abs(beatsFromStart - Math.round(beatsFromStart))).toBeLessThan(0.001);
});
});
it('uses style pools with multiple notes instead of one repeating key', () => {
const { engine, notes } = makeEngine();
renderBars(engine, 1, 16);
expect(new Set(notes.map((note) => note.midi)).size).toBeGreaterThan(3);
});
it('changes musical style over time without a color change', () => {
const { engine, notes } = makeEngine();
renderBars(engine, 1, 32);
const styleWindows = [
notes.filter((note) => note.startTime >= 0 && note.startTime < 8),
notes.filter((note) => note.startTime >= 8 && note.startTime < 16),
notes.filter((note) => note.startTime >= 16 && note.startTime < 24),
];
const averageMidiByWindow = styleWindows.map((windowNotes) =>
Math.round(average(windowNotes.map((note) => note.midi)))
);
const averagePanByWindow = styleWindows.map((windowNotes) =>
Number(average(windowNotes.map((note) => note.pan)).toFixed(2))
);
expect(styleWindows.every((windowNotes) => windowNotes.length > 0)).toBe(true);
expect(new Set(averageMidiByWindow).size).toBeGreaterThan(1);
expect(new Set(averagePanByWindow).size).toBeGreaterThan(1);
});
it('starts a fading brush phrase layer with each new brush gesture', () => {
const baseline = makeEngine();
const layered = makeEngine();
const now = 4;
baseline.engine.renderLookahead({
vibe: VIBE_PRESETS[0],
now,
activity: 0.35,
lookaheadSeconds: 12,
});
layered.engine.beginGesture();
layered.engine.recordStroke({
vibe: VIBE_PRESETS[0],
now,
activity: 0.85,
});
layered.engine.renderLookahead({
vibe: VIBE_PRESETS[0],
now,
activity: 0.35,
lookaheadSeconds: 12,
});
const earlyExtra =
countNotesBetween(layered.notes, now + 1, now + 5) -
countNotesBetween(baseline.notes, now + 1, now + 5);
const lateExtra =
countNotesBetween(layered.notes, now + 10.5, now + 12) -
countNotesBetween(baseline.notes, now + 10.5, now + 12);
expect(earlyExtra).toBeGreaterThan(2);
expect(lateExtra).toBe(0);
});
it('plays one immediate touch note and throttles later stroke accents', () => {
const { engine, notes } = makeEngine();
const now = 4;
engine.beginGesture();
engine.recordStroke({
vibe: VIBE_PRESETS[0],
now,
activity: 0.9,
});
engine.recordStroke({
vibe: VIBE_PRESETS[0],
now: now + 1,
activity: 0.95,
});
expect(notes).toHaveLength(1);
expect(notes[0].startTime).toBe(now);
engine.recordStroke({
vibe: VIBE_PRESETS[0],
now: now + 6,
activity: 0.95,
});
expect(notes).toHaveLength(2);
expect(new Set(notes.map((note) => note.midi)).size).toBeGreaterThan(1);
});
it('is deterministic for the same musical inputs', () => {
const first = makeEngine();
const second = makeEngine();
renderBars(first.engine, 0.78, 16);
renderBars(second.engine, 0.78, 16);
expect(second.notes).toEqual(first.notes);
});
it('does not duplicate notes across overlapping lookahead windows', () => {
const { engine, notes } = makeEngine();
engine.renderLookahead({
vibe: VIBE_PRESETS[0],
now: 0,
activity: 0.72,
lookaheadSeconds: getBeatSeconds() * 2,
});
engine.renderLookahead({
vibe: VIBE_PRESETS[0],
now: getBeatSeconds() * 0.5,
activity: 0.72,
lookaheadSeconds: getBeatSeconds() * 2,
});
const noteKeys = notes.map(getNoteKey);
expect(new Set(noteKeys).size).toBe(noteKeys.length);
});
it('keeps generated notes inside the audio contract', () => {
const { engine, notes } = makeEngine();
VIBE_PRESETS.forEach((vibe) => {
engine.cue(0);
engine.renderLookahead({
vibe,
now: 0,
activity: 1,
lookaheadSeconds: getBeatSeconds() * getBeatsPerBar() * 4,
});
});
notes.forEach((note) => {
expect(Number.isFinite(note.startTime)).toBe(true);
expect(note.midi).toBeGreaterThanOrEqual(21);
expect(note.midi).toBeLessThanOrEqual(108);
expect(note.velocity).toBeGreaterThan(0);
expect(note.velocity).toBeLessThanOrEqual(0.4);
expect(note.pan).toBeGreaterThanOrEqual(-1);
expect(note.pan).toBeLessThanOrEqual(1);
expect(note.durationSeconds).toBeGreaterThan(0);
expect(note.lowpassHz ?? gardenAudioConfig.piano.lowpassHz).toBeGreaterThanOrEqual(
gardenAudioConfig.piano.lowpassMinHz
);
expect(note.lowpassHz ?? gardenAudioConfig.piano.lowpassHz).toBeLessThanOrEqual(
gardenAudioConfig.piano.lowpassMaxHz
);
});
});
});

63
src/audio/generative-piano.ts
View file

@ -1,18 +1,28 @@
import { clamp, clamp01 } from '../utils/clamp';
import { VibePreset } from '../vibes';
import {
import { clamp, clamp01 } from '../utils/math';
import type { VibePreset } from '../vibes';
import type {
GardenAudioChord,
GardenAudioConfig,
GardenAudioRegister,
GardenAudioStylePool,
GardenAudioVibeProfile,
} from './garden-audio-config';
import {
degreeToSemitone,
getChordIntervals,
getVibeProfile,
PITCH_SEMITONES_PER_OCTAVE,
} from './garden-audio-music';
import { PianoNote } from './garden-audio-types';
import {
GENERATIVE_LOOKAHEAD_SECONDS,
GENERATIVE_START_DELAY_SECONDS,
PIANO_SCHEDULE_AHEAD_SECONDS,
} from './garden-audio-scheduling';
import type { PianoNote } from './garden-audio-types';
import {
generativePianoTuning,
styleVoices,
type GardenAudioRegister,
type GardenAudioStylePool,
} from './generative-piano-tuning';
type GardenAudioStyleIndex = 0 | 1 | 2;
@ -87,8 +97,8 @@ export class GenerativePianoEngine {
private readonly playNote: (note: PianoNote) => void
) {}
private get generation(): GardenAudioConfig['generativePiano'] {
return this.config.generativePiano;
private get generation(): typeof generativePianoTuning {
return generativePianoTuning;
}
public prime(now: number): void {
@ -190,7 +200,7 @@ export class GenerativePianoEngine {
vibe,
now,
activity,
lookaheadSeconds = this.config.rhythm.lookaheadSeconds,
lookaheadSeconds = GENERATIVE_LOOKAHEAD_SECONDS,
}: RenderLookaheadRequest): void {
this.prime(now);
this.skipLateBeats(now);
@ -418,7 +428,7 @@ export class GenerativePianoEngine {
velocity:
(this.generation.supportNote.velocityBase +
expression * this.generation.supportNote.velocityExpressionWeight) *
this.config.styleVoices[styleIndex].velocityMultiplier,
styleVoices[styleIndex].velocityMultiplier,
startTime,
durationSeconds:
this.generation.supportNote.durationBaseSeconds +
@ -464,7 +474,7 @@ export class GenerativePianoEngine {
velocity:
(this.generation.textureNote.velocityBase +
expression * this.generation.textureNote.velocityExpressionWeight) *
this.config.styleVoices[styleIndex].velocityMultiplier,
styleVoices[styleIndex].velocityMultiplier,
startTime,
durationSeconds:
this.generation.textureNote.durationBaseSeconds +
@ -511,7 +521,7 @@ export class GenerativePianoEngine {
velocity:
(this.generation.gestureAccent.velocityBase +
strength * this.generation.gestureAccent.velocityStrengthWeight) *
this.config.styleVoices[styleIndex].velocityMultiplier,
styleVoices[styleIndex].velocityMultiplier,
startTime,
durationSeconds:
this.generation.gestureAccent.durationBaseSeconds +
@ -560,7 +570,7 @@ export class GenerativePianoEngine {
velocity:
(this.generation.touchNote.velocityBase +
strength * this.generation.touchNote.velocityStrengthWeight) *
this.config.styleVoices[styleIndex].velocityMultiplier,
styleVoices[styleIndex].velocityMultiplier,
startTime: now,
durationSeconds:
this.generation.touchNote.durationBaseSeconds +
@ -661,7 +671,7 @@ export class GenerativePianoEngine {
lookaheadEnd: number;
activity: number;
}): void {
const earliestStart = now + this.config.piano.scheduleAheadSeconds;
const earliestStart = now + PIANO_SCHEDULE_AHEAD_SECONDS;
this.nextBrushStreamStep ??= 0;
this.pruneBrushStreamNoteCounts(this.getGlobalBarIndex(now) - 1);
@ -770,7 +780,7 @@ export class GenerativePianoEngine {
velocity:
(this.generation.brushStream.velocityBase +
intensity * this.generation.brushStream.velocityIntensityWeight) *
this.config.styleVoices[styleIndex].velocityMultiplier,
styleVoices[styleIndex].velocityMultiplier,
startTime,
durationSeconds,
pan,
@ -803,7 +813,7 @@ export class GenerativePianoEngine {
velocity:
(this.generation.brushStreamEcho.velocityBase +
intensity * this.generation.brushStreamEcho.velocityIntensityWeight) *
this.config.styleVoices[styleIndex].velocityMultiplier,
styleVoices[styleIndex].velocityMultiplier,
startTime: startTime + this.generation.brushMotifCanonDelaySeconds,
durationSeconds: Math.max(
this.generation.brushStreamEcho.durationMinSeconds,
@ -872,8 +882,8 @@ export class GenerativePianoEngine {
: intensity >= this.generation.brushStream.intenseThreshold
? this.generation.brushStreamIntenseIntervalBeats
: intensity >= this.generation.brushStream.activeThreshold
? this.generation.brushStreamActiveIntervalBeats
: this.generation.brushStreamIdleIntervalBeats;
? this.generation.brushStreamActiveIntervalBeats
: this.generation.brushStreamIdleIntervalBeats;
return Math.max(1, Math.round(intervalBeats * this.config.rhythm.stepsPerBeat));
}
@ -913,7 +923,7 @@ export class GenerativePianoEngine {
pool: GardenAudioStylePool;
styleIndex: GardenAudioStyleIndex;
}): Array<number> {
const styleOffset = this.config.styleVoices[styleIndex].scaleDegreeOffset;
const styleOffset = styleVoices[styleIndex].scaleDegreeOffset;
if (!layer || layer.motifOffsets.length === 0) {
return this.rotate(pool.scaleDegrees, this.brushStreamNoteIndex + styleOffset);
}
@ -990,10 +1000,7 @@ export class GenerativePianoEngine {
octave <= this.generation.candidateOctaveSearch.max;
octave += 1
) {
const midi =
pitchSource.baseMidi +
offset +
octave * this.config.piano.pitchSemitonesPerOctave;
const midi = pitchSource.baseMidi + offset + octave * PITCH_SEMITONES_PER_OCTAVE;
if (midi >= register.midiMin && midi <= register.midiMax) {
const roundedMidi = Math.round(midi);
candidates.push({
@ -1080,7 +1087,7 @@ export class GenerativePianoEngine {
private getStylePan(styleIndex: GardenAudioStyleIndex): number {
const pool = this.generation.stylePools[styleIndex];
const styleVoice = this.config.styleVoices[styleIndex];
const styleVoice = styleVoices[styleIndex];
return clamp(
pool.pan + styleVoice.panOffset * this.generation.stylePanOffsetScale,
-1,
@ -1113,7 +1120,7 @@ export class GenerativePianoEngine {
return;
}
const earliestStart = now + this.config.piano.scheduleAheadSeconds;
const earliestStart = now + PIANO_SCHEDULE_AHEAD_SECONDS;
if (this.getTimeForStep(this.nextBeatStep) >= earliestStart) {
return;
}
@ -1152,7 +1159,7 @@ export class GenerativePianoEngine {
private getTimeForStep(stepIndex: number): number {
return (
(this.timelineStartedAt ?? 0) +
this.config.startDelaySeconds +
GENERATIVE_START_DELAY_SECONDS +
stepIndex * this.getStepDurationSeconds()
);
}
@ -1161,7 +1168,7 @@ export class GenerativePianoEngine {
const timelineStartedAt = this.timelineStartedAt ?? startTime;
const elapsedSeconds = Math.max(
0,
startTime - timelineStartedAt - this.config.startDelaySeconds
startTime - timelineStartedAt - GENERATIVE_START_DELAY_SECONDS
);
return Math.floor(elapsedSeconds / this.getStepDurationSeconds());
}
@ -1170,7 +1177,7 @@ export class GenerativePianoEngine {
const timelineStartedAt = this.timelineStartedAt ?? startTime;
const elapsedSeconds = Math.max(
0,
startTime - timelineStartedAt - this.config.startDelaySeconds
startTime - timelineStartedAt - GENERATIVE_START_DELAY_SECONDS
);
return Math.max(
0,

49
src/audio/noise-burst-player.ts
View file

@ -1,12 +1,18 @@
import type { GardenAudioConfig } from './garden-audio-config';
import { GardenAudioGraph } from './garden-audio-graph';
import { NoiseBurst } from './garden-audio-types';
import { createAudioPanNode } from './audio-pan-node';
import type { GardenAudioGraph } from './garden-audio-graph';
import type { NoiseBurst } from './garden-audio-types';
const noiseBurstTuning = {
attackSeconds: 0.004,
filterQ: 1.4,
offsetRandomSeconds: 0.4,
scheduleAheadSeconds: 0.002,
silentGain: 0.0001,
filterType: 'bandpass' as BiquadFilterType,
};
export class NoiseBurstPlayer {
public constructor(
private readonly config: GardenAudioConfig,
private readonly graph: GardenAudioGraph
) {}
public constructor(private readonly graph: GardenAudioGraph) {}
public play({ startTime, durationSeconds, gain, filterHz, pan }: NoiseBurst): void {
const { context, eventBus, noiseBus, noiseBuffer } = this.graph;
@ -16,35 +22,30 @@ export class NoiseBurstPlayer {
}
const scheduledStart = Math.max(
context.currentTime + this.config.noiseBurst.scheduleAheadSeconds,
context.currentTime + noiseBurstTuning.scheduleAheadSeconds,
startTime
);
const source = context.createBufferSource();
const filter = context.createBiquadFilter();
const envelope = context.createGain();
const panner = context.createStereoPanner();
const panNode = createAudioPanNode(context, pan, scheduledStart);
const stopAt = scheduledStart + durationSeconds;
source.buffer = noiseBuffer;
filter.type = this.config.noiseBurst.filterType;
filter.type = noiseBurstTuning.filterType;
filter.frequency.setValueAtTime(filterHz, scheduledStart);
filter.Q.value = this.config.noiseBurst.filterQ;
envelope.gain.setValueAtTime(this.config.noiseBurst.silentGain, scheduledStart);
filter.Q.value = noiseBurstTuning.filterQ;
envelope.gain.setValueAtTime(noiseBurstTuning.silentGain, scheduledStart);
envelope.gain.exponentialRampToValueAtTime(
Math.max(this.config.noiseBurst.silentGain, gain),
scheduledStart + this.config.noiseBurst.attackSeconds
Math.max(noiseBurstTuning.silentGain, gain),
scheduledStart + noiseBurstTuning.attackSeconds
);
envelope.gain.exponentialRampToValueAtTime(this.config.noiseBurst.silentGain, stopAt);
panner.pan.setValueAtTime(pan, scheduledStart);
envelope.gain.exponentialRampToValueAtTime(noiseBurstTuning.silentGain, stopAt);
source.connect(filter);
filter.connect(envelope);
envelope.connect(panner);
panner.connect(outputBus);
source.start(
scheduledStart,
Math.random() * this.config.noiseBurst.offsetRandomSeconds
);
envelope.connect(panNode.input);
panNode.output.connect(outputBus);
source.start(scheduledStart, Math.random() * noiseBurstTuning.offsetRandomSeconds);
source.stop(stopAt);
source.addEventListener(
'ended',
@ -52,7 +53,7 @@ export class NoiseBurstPlayer {
source.disconnect();
filter.disconnect();
envelope.disconnect();
panner.disconnect();
panNode.disconnect();
},
{ once: true }
);

167
src/audio/piano-sampler.test.ts
View file

@ -1,167 +0,0 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { gardenAudioConfig } from './garden-audio-config';
import type { GardenAudioGraph } from './garden-audio-graph';
import type { PianoSampler } from './piano-sampler';
const calls = {
bufferSourcesStarted: 0,
};
const sampleCount = 30;
class FakeAudioParam {
public value = 0;
public setTargetAtTime = vi.fn();
public setValueAtTime = vi.fn();
public exponentialRampToValueAtTime = vi.fn();
public cancelScheduledValues = vi.fn();
}
class FakeAudioNode {
public readonly gain = new FakeAudioParam();
public readonly frequency = new FakeAudioParam();
public readonly playbackRate = new FakeAudioParam();
public readonly Q = new FakeAudioParam();
public readonly pan = new FakeAudioParam();
public buffer: AudioBuffer | null = null;
public type = '';
public addEventListener = vi.fn();
public connect = vi.fn();
public disconnect = vi.fn();
public start = vi.fn();
public stop = vi.fn();
}
class FakeAudioContext {
public readonly currentTime = 1;
public readonly decodeAudioData = vi.fn(async () => ({}) as AudioBuffer);
public createGain(): GainNode {
return new FakeAudioNode() as unknown as GainNode;
}
public createBiquadFilter(): BiquadFilterNode {
return new FakeAudioNode() as unknown as BiquadFilterNode;
}
public createStereoPanner(): StereoPannerNode {
return new FakeAudioNode() as unknown as StereoPannerNode;
}
public createBufferSource(): AudioBufferSourceNode {
const node = new FakeAudioNode() as unknown as AudioBufferSourceNode & {
start: () => void;
stop: () => void;
};
node.start = vi.fn(() => {
calls.bufferSourcesStarted += 1;
});
node.stop = vi.fn();
return node;
}
}
const makeSampler = async (context: AudioContext): Promise<PianoSampler> => {
const { PianoSampler } = await import('./piano-sampler');
const eventBus = new FakeAudioNode() as unknown as GainNode;
const graph = {
context,
delayInput: null,
eventBus,
getPianoBus: vi.fn(() => eventBus),
} as unknown as GardenAudioGraph;
return new PianoSampler(gardenAudioConfig, graph);
};
describe('PianoSampler', () => {
beforeEach(() => {
calls.bufferSourcesStarted = 0;
vi.resetModules();
});
afterEach(() => {
vi.unstubAllGlobals();
});
it('loads every piano sample before playback', async () => {
const context = new FakeAudioContext() as unknown as AudioContext;
const sampler = await makeSampler(context);
const fetch = vi.fn(async () => {
return {
arrayBuffer: async () => new ArrayBuffer(8),
ok: true,
} as Response;
});
vi.stubGlobal('fetch', fetch);
await sampler.load(context);
sampler.play({
durationSeconds: 0.2,
midi: 60,
pan: 0,
startTime: context.currentTime,
velocity: 0.5,
});
expect(fetch).toHaveBeenCalledTimes(sampleCount);
expect(context.decodeAudioData).toHaveBeenCalledTimes(sampleCount);
expect(calls.bufferSourcesStarted).toBe(1);
});
it('only queues a piano load when the sampler is idle', async () => {
const context = new FakeAudioContext() as unknown as AudioContext;
const sampler = await makeSampler(context);
const fetch = vi.fn(async () => {
return {
arrayBuffer: async () => new ArrayBuffer(8),
ok: true,
} as Response;
});
vi.stubGlobal('fetch', fetch);
const firstLoad = sampler.loadIfIdle(context);
const secondLoad = sampler.loadIfIdle(context);
expect(firstLoad).toBeInstanceOf(Promise);
expect(secondLoad).toBeNull();
await firstLoad;
expect(sampler.loadIfIdle(context)).toBeNull();
expect(fetch).toHaveBeenCalledTimes(sampleCount);
});
it('allows loading to be retried after a load failure', async () => {
const context = new FakeAudioContext() as unknown as AudioContext;
const sampler = await makeSampler(context);
const fetch = vi
.fn()
.mockRejectedValueOnce(new Error('load failed'))
.mockResolvedValue({
arrayBuffer: async () => new ArrayBuffer(8),
ok: true,
} as Response);
vi.stubGlobal('fetch', fetch);
await expect(sampler.loadIfIdle(context)).rejects.toThrow('load failed');
await expect(sampler.loadIfIdle(context)).resolves.toBeUndefined();
expect(fetch).toHaveBeenCalledTimes(sampleCount * 2);
});
it('stays silent when no decoded sample is available', () => {
const context = new FakeAudioContext() as unknown as AudioContext;
return makeSampler(context).then((sampler) => {
sampler.play({
durationSeconds: 0.2,
midi: 60,
pan: 0,
startTime: context.currentTime,
velocity: 0.5,
});
expect(calls.bufferSourcesStarted).toBe(0);
});
});
});

222
src/audio/piano-sampler.ts
View file

@ -1,11 +1,32 @@
import { clamp, clamp01 } from '../utils/clamp';
import { GardenAudioConfig } from './garden-audio-config';
import { GardenAudioGraph } from './garden-audio-graph';
import { ActivePianoVoice, LoadedPianoSample, PianoNote } from './garden-audio-types';
import { clamp, clamp01 } from '../utils/math';
import { createAudioPanNode } from './audio-pan-node';
import type { GardenAudioConfig } from './garden-audio-config';
import type { GardenAudioGraph } from './garden-audio-graph';
import { PITCH_SEMITONES_PER_OCTAVE } from './garden-audio-music';
import { PIANO_SCHEDULE_AHEAD_SECONDS } from './garden-audio-scheduling';
import type {
ActivePianoVoice,
LoadedPianoSample,
PianoNote,
} from './garden-audio-types';
import { getLoadedPianoSamples, loadPianoSamples } from './piano-samples';
type PianoLoadState = 'idle' | 'loading' | 'loaded';
const pianoSamplerTuning = {
filterType: 'lowpass' as BiquadFilterType,
filterQ: 0.7,
minDurationSeconds: 0.08,
minFadeSeconds: 0.08,
minGain: 0.0001,
synthGainScale: 0.34,
synthMaxDurationSeconds: 1.8,
synthOscillatorType: 'triangle' as OscillatorType,
tailStopExtraSeconds: 0.05,
voiceStealFadeSeconds: 0.025,
voiceStealStopSeconds: 0.05,
};
export class PianoSampler {
private loadState: PianoLoadState = 'idle';
private sampleLoadPromise: Promise<void> | null = null;
@ -34,7 +55,9 @@ export class PianoSampler {
}
this.loadState = 'loading';
this.sampleLoadPromise = loadPianoSamples(context)
this.sampleLoadPromise = loadPianoSamples(context, undefined, {
forceReload: true,
})
.then((samples) => {
this.setSamples(samples);
this.loadState = 'loaded';
@ -74,16 +97,26 @@ export class PianoSampler {
const sample = this.findNearestSample(midi);
if (!sample) {
this.playSynthFallback({
midi,
velocity,
startTime,
durationSeconds,
pan,
role,
delaySend,
lowpassHz,
});
return;
}
const scheduledStart = Math.max(
context.currentTime + this.config.piano.scheduleAheadSeconds,
context.currentTime + PIANO_SCHEDULE_AHEAD_SECONDS,
startTime
);
const noteVelocity = clamp01(velocity);
const noteGainValue = Math.max(
this.config.piano.minGain,
pianoSamplerTuning.minGain,
this.config.piano.gain * noteVelocity
);
const sustainSeconds =
@ -91,14 +124,14 @@ export class PianoSampler {
(this.config.piano.sustainBase +
noteVelocity * this.config.piano.sustainVelocityRange);
const sustainAt =
scheduledStart + Math.max(this.config.piano.minDurationSeconds, durationSeconds);
scheduledStart + Math.max(pianoSamplerTuning.minDurationSeconds, durationSeconds);
const releaseAt = sustainAt + sustainSeconds;
const releaseSeconds = this.config.piano.releaseSeconds;
const stopAt = releaseAt + releaseSeconds;
const source = context.createBufferSource();
const filter = context.createBiquadFilter();
const gain = context.createGain();
const panner = context.createStereoPanner();
const panNode = createAudioPanNode(context, pan, scheduledStart);
let sendGain: GainNode | null = null;
this.trimActiveVoices(scheduledStart);
@ -111,45 +144,46 @@ export class PianoSampler {
source.buffer = sample.buffer;
source.playbackRate.setValueAtTime(
Math.pow(2, (midi - sample.midi) / this.config.piano.pitchSemitonesPerOctave),
Math.pow(2, (midi - sample.midi) / PITCH_SEMITONES_PER_OCTAVE),
scheduledStart
);
filter.type = this.config.piano.filterType;
filter.type = pianoSamplerTuning.filterType;
filter.frequency.setValueAtTime(
clamp(lowpassHz, this.config.piano.lowpassMinHz, this.config.piano.lowpassMaxHz),
scheduledStart
);
filter.Q.value = this.config.piano.filterQ;
gain.gain.setValueAtTime(this.config.piano.minGain, scheduledStart);
filter.Q.value = pianoSamplerTuning.filterQ;
gain.gain.setValueAtTime(pianoSamplerTuning.minGain, scheduledStart);
gain.gain.exponentialRampToValueAtTime(
noteGainValue,
scheduledStart + this.config.piano.gainAttackSeconds
);
gain.gain.setTargetAtTime(
Math.max(this.config.piano.minGain, noteGainValue * this.config.piano.sustainLevel),
Math.max(
pianoSamplerTuning.minGain,
noteGainValue * this.config.piano.sustainLevel
),
sustainAt,
Math.max(
this.config.piano.minFadeSeconds,
pianoSamplerTuning.minFadeSeconds,
sustainSeconds * this.config.piano.sustainBase
)
);
gain.gain.setTargetAtTime(this.config.piano.minGain, releaseAt, releaseSeconds);
panner.pan.setValueAtTime(clamp(pan, -1, 1), scheduledStart);
gain.gain.setTargetAtTime(pianoSamplerTuning.minGain, releaseAt, releaseSeconds);
source.connect(filter);
filter.connect(gain);
gain.connect(panner);
panner.connect(eventBus);
gain.connect(panNode.input);
panNode.output.connect(eventBus);
if (delayInput && delaySend > 0) {
sendGain = context.createGain();
sendGain.gain.value = delaySend;
panner.connect(sendGain);
panNode.output.connect(sendGain);
sendGain.connect(delayInput);
}
source.start(scheduledStart);
source.stop(stopAt + this.config.piano.tailStopExtraSeconds);
source.stop(stopAt + pianoSamplerTuning.tailStopExtraSeconds);
this.activeVoices.push({ gain, source, stopAt });
source.addEventListener(
@ -158,7 +192,7 @@ export class PianoSampler {
source.disconnect();
filter.disconnect();
gain.disconnect();
panner.disconnect();
panNode.disconnect();
sendGain?.disconnect();
this.activeVoices = this.activeVoices.filter((voice) => voice.gain !== gain);
},
@ -181,6 +215,22 @@ export class PianoSampler {
this.activeVoices = [];
}
public fadeAll(fadeSeconds: number): void {
const context = this.graph.context;
if (!context) {
this.activeVoices = [];
return;
}
const now = context.currentTime;
const fadeDurationSeconds = Math.max(pianoSamplerTuning.minFadeSeconds, fadeSeconds);
this.trimActiveVoices(now);
this.activeVoices.forEach((voice) => {
this.fadeVoice(voice, now, fadeDurationSeconds);
});
}
public reset(): void {
this.loadState = 'idle';
this.sampleLoadPromise = null;
@ -198,18 +248,114 @@ export class PianoSampler {
);
}
private playSynthFallback({
midi,
velocity,
startTime,
durationSeconds,
pan,
role,
delaySend = 0,
lowpassHz = this.config.piano.lowpassHz,
}: PianoNote): void {
const { context, delayInput } = this.graph;
const eventBus = this.graph.getPianoBus(role);
if (!context || !eventBus) {
return;
}
const scheduledStart = Math.max(
context.currentTime + PIANO_SCHEDULE_AHEAD_SECONDS,
startTime
);
const noteVelocity = clamp01(velocity);
const noteGainValue = Math.max(
pianoSamplerTuning.minGain,
this.config.piano.gain * noteVelocity * pianoSamplerTuning.synthGainScale
);
const releaseAt =
scheduledStart +
clamp(
durationSeconds + this.config.piano.sustainSeconds * 0.5,
pianoSamplerTuning.minDurationSeconds,
pianoSamplerTuning.synthMaxDurationSeconds
);
const stopAt = releaseAt + this.config.piano.releaseSeconds;
const source = context.createOscillator();
const filter = context.createBiquadFilter();
const gain = context.createGain();
const panNode = createAudioPanNode(context, pan, scheduledStart);
let sendGain: GainNode | null = null;
this.trimActiveVoices(scheduledStart);
while (this.activeVoices.length >= this.config.piano.maxVoices) {
const oldest = this.activeVoices.shift();
if (oldest) {
this.stopVoice(oldest, scheduledStart);
}
}
source.type = pianoSamplerTuning.synthOscillatorType;
source.frequency.setValueAtTime(getMidiFrequency(midi), scheduledStart);
filter.type = pianoSamplerTuning.filterType;
filter.frequency.setValueAtTime(
clamp(lowpassHz, this.config.piano.lowpassMinHz, this.config.piano.lowpassMaxHz),
scheduledStart
);
filter.Q.value = pianoSamplerTuning.filterQ;
gain.gain.setValueAtTime(pianoSamplerTuning.minGain, scheduledStart);
gain.gain.exponentialRampToValueAtTime(
noteGainValue,
scheduledStart + this.config.piano.gainAttackSeconds
);
gain.gain.setTargetAtTime(
pianoSamplerTuning.minGain,
releaseAt,
this.config.piano.releaseSeconds
);
source.connect(filter);
filter.connect(gain);
gain.connect(panNode.input);
panNode.output.connect(eventBus);
if (delayInput && delaySend > 0) {
sendGain = context.createGain();
sendGain.gain.value = delaySend;
panNode.output.connect(sendGain);
sendGain.connect(delayInput);
}
source.start(scheduledStart);
source.stop(stopAt + pianoSamplerTuning.tailStopExtraSeconds);
this.activeVoices.push({ gain, source, stopAt });
source.addEventListener(
'ended',
() => {
source.disconnect();
filter.disconnect();
gain.disconnect();
panNode.disconnect();
sendGain?.disconnect();
this.activeVoices = this.activeVoices.filter((voice) => voice.gain !== gain);
},
{ once: true }
);
}
private trimActiveVoices(now: number): void {
this.activeVoices = this.activeVoices.filter((voice) => voice.stopAt > now);
}
private stopVoice(voice: ActivePianoVoice, now: number): void {
const stopAt = now + this.config.piano.voiceStealStopSeconds;
const stopAt = now + pianoSamplerTuning.voiceStealStopSeconds;
voice.gain.gain.cancelScheduledValues(now);
voice.gain.gain.setTargetAtTime(
this.config.piano.minGain,
pianoSamplerTuning.minGain,
now,
this.config.piano.voiceStealFadeSeconds
pianoSamplerTuning.voiceStealFadeSeconds
);
voice.stopAt = stopAt;
try {
@ -219,7 +365,31 @@ export class PianoSampler {
}
}
private fadeVoice(
voice: ActivePianoVoice,
now: number,
fadeDurationSeconds: number
): void {
const stopAt = Math.min(voice.stopAt, now + fadeDurationSeconds);
voice.gain.gain.cancelScheduledValues(now);
voice.gain.gain.setTargetAtTime(
pianoSamplerTuning.minGain,
now,
Math.max(0.001, fadeDurationSeconds / 4)
);
voice.stopAt = stopAt;
try {
voice.source.stop(stopAt + pianoSamplerTuning.tailStopExtraSeconds);
} catch {
// The voice may already have ended; either way it is fading out of the mix.
}
}
private setSamples(samples: Array<LoadedPianoSample>): void {
this.samples = samples.slice().sort((a, b) => a.midi - b.midi);
}
}
const getMidiFrequency = (midi: number): number =>
440 * Math.pow(2, (midi - 69) / PITCH_SEMITONES_PER_OCTAVE);

97
src/audio/piano-samples.ts
View file

@ -1,4 +1,3 @@
import { gardenAudioConfig } from './garden-audio-config';
import type { LoadedPianoSample } from './garden-audio-types';
interface PianoSampleDefinition {
@ -45,21 +44,36 @@ const sampleFiles: Array<[fileName: string, midi: number]> = [
['C8v12.m4a', 108],
];
const sampleBaseUrl = `${import.meta.env.BASE_URL}audio/`;
const preloadDecode = {
channels: 2,
frames: 128,
sampleRateHz: 48_000,
};
const pianoSampleDefinitions: Array<PianoSampleDefinition> = sampleFiles
.map(([fileName, midi]) => ({
midi,
url: `${gardenAudioConfig.piano.sampleBaseUrl}${fileName}`,
url: `${sampleBaseUrl}${fileName}`,
}))
.sort((a, b) => a.midi - b.midi);
let loadedPianoSamples: Array<LoadedPianoSample> | null = null;
let pianoSampleLoadPromise: Promise<Array<LoadedPianoSample>> | null = null;
interface PianoSampleLoadOptions {
forceReload?: boolean;
}
const sampleLoadTuning = {
concurrency: 4,
sampleTimeoutMs: 15_000,
};
export const preloadPianoSamples = (
onProgress?: (progress: PianoSampleLoadProgress) => void
): Promise<Array<LoadedPianoSample>> => {
const OfflineAudioContextConstructor =
globalThis.OfflineAudioContext ?? globalThis.webkitOfflineAudioContext;
const OfflineAudioContextConstructor = globalThis.OfflineAudioContext;
if (!OfflineAudioContextConstructor) {
return Promise.reject(
@ -68,16 +82,17 @@ export const preloadPianoSamples = (
}
const decodeContext = new OfflineAudioContextConstructor(
gardenAudioConfig.piano.preloadDecode.channels,
gardenAudioConfig.piano.preloadDecode.frames,
gardenAudioConfig.piano.preloadDecode.sampleRateHz
preloadDecode.channels,
preloadDecode.frames,
preloadDecode.sampleRateHz
);
return loadPianoSamples(decodeContext, onProgress);
};
export const loadPianoSamples = (
decodeContext: BaseAudioContext,
onProgress?: (progress: PianoSampleLoadProgress) => void
onProgress?: (progress: PianoSampleLoadProgress) => void,
options: PianoSampleLoadOptions = {}
): Promise<Array<LoadedPianoSample>> => {
if (loadedPianoSamples) {
onProgress?.({
@ -87,7 +102,7 @@ export const loadPianoSamples = (
return Promise.resolve([...loadedPianoSamples]);
}
if (pianoSampleLoadPromise) {
if (pianoSampleLoadPromise && options.forceReload !== true) {
return pianoSampleLoadPromise;
}
@ -95,16 +110,27 @@ export const loadPianoSamples = (
const totalCount = pianoSampleDefinitions.length;
onProgress?.({ loadedCount, totalCount });
pianoSampleLoadPromise = Promise.all(
pianoSampleDefinitions.map(async (sample) => {
const loadedSample = await loadPianoSample(decodeContext, sample);
loadedCount += 1;
onProgress?.({ loadedCount, totalCount, sample });
return loadedSample;
})
pianoSampleLoadPromise = loadPianoSampleBatch(
pianoSampleDefinitions,
async (sample) => {
try {
return await withTimeout(
loadPianoSample(decodeContext, sample),
sampleLoadTuning.sampleTimeoutMs
);
} finally {
loadedCount += 1;
onProgress?.({ loadedCount, totalCount, sample });
}
}
).then(
(samples) => {
loadedPianoSamples = samples.slice().sort((a, b) => a.midi - b.midi);
loadedPianoSamples = samples
.filter((sample): sample is LoadedPianoSample => sample !== null)
.sort((a, b) => a.midi - b.midi);
if (loadedPianoSamples.length === 0) {
throw new Error('Unable to load any piano samples.');
}
return [...loadedPianoSamples];
},
(error: unknown) => {
@ -133,6 +159,43 @@ const loadPianoSample = async (
return { midi: sample.midi, buffer };
};
const loadPianoSampleBatch = async (
samples: Array<PianoSampleDefinition>,
loadSample: (
sample: PianoSampleDefinition
) => Promise<LoadedPianoSample | null>
): Promise<Array<LoadedPianoSample | null>> => {
const results: Array<LoadedPianoSample | null> = [];
for (let index = 0; index < samples.length; index += sampleLoadTuning.concurrency) {
const batch = samples.slice(index, index + sampleLoadTuning.concurrency);
const batchResults = await Promise.all(
batch.map((sample) => loadSample(sample).catch(() => null))
);
results.push(...batchResults);
}
return results;
};
const withTimeout = <T>(promise: Promise<T>, timeoutMs: number): Promise<T> =>
new Promise((resolve, reject) => {
const timeout = globalThis.setTimeout(() => {
reject(new Error('Timed out while loading a piano sample.'));
}, timeoutMs);
promise.then(
(value) => {
globalThis.clearTimeout(timeout);
resolve(value);
},
(error: unknown) => {
globalThis.clearTimeout(timeout);
reject(error);
}
);
});
const decodeAudioData = (
decodeContext: BaseAudioContext,
audioData: ArrayBuffer