0e73fd1348
* Resolved an issue with the ESP8266 where gamma correction would be performed twice. Changed GAMMA_CORRECTION to SOFTWARE_GAMMA_CORRECTION to make a distinction between software and firmware gamma correction. The ESP8266 does firmware gamma correction and dithering, while the Raspberry Pi uses slightly more inferior software gamma correction. Changed the software gamma table to match the gamma table used in the ESP8266 firmware. * Improved the spectrum visualization by using one of the color channels to visualize the absolute value of the temporal derivative of the spectrum. Also added a feature to reject the "common mode" spectral components, which is analogous to the spectral DC component. * Signficantly improved the GUI and added a frequency adjustment slider. Adjusting the frequency range has a big impact on the visualization output. Recommend using a high frequency range (something like 4 kHz - 10 kHz) when running the scrol visualization.
340 lines
12 KiB
Python
340 lines
12 KiB
Python
from __future__ import print_function
|
|
from __future__ import division
|
|
import time
|
|
import numpy as np
|
|
from scipy.ndimage.filters import gaussian_filter1d
|
|
import config
|
|
import microphone
|
|
import dsp
|
|
import led
|
|
|
|
_time_prev = time.time() * 1000.0
|
|
"""The previous time that the frames_per_second() function was called"""
|
|
|
|
_fps = dsp.ExpFilter(val=config.FPS, alpha_decay=0.002, alpha_rise=0.002)
|
|
"""The low-pass filter used to estimate frames-per-second"""
|
|
|
|
|
|
def frames_per_second():
|
|
"""Return the estimated frames per second
|
|
|
|
Returns the current estimate for frames-per-second (FPS).
|
|
FPS is estimated by measured the amount of time that has elapsed since
|
|
this function was previously called. The FPS estimate is low-pass filtered
|
|
to reduce noise.
|
|
|
|
This function is intended to be called one time for every iteration of
|
|
the program's main loop.
|
|
|
|
Returns
|
|
-------
|
|
fps : float
|
|
Estimated frames-per-second. This value is low-pass filtered
|
|
to reduce noise.
|
|
"""
|
|
global _time_prev, _fps
|
|
time_now = time.time() * 1000.0
|
|
dt = time_now - _time_prev
|
|
_time_prev = time_now
|
|
if dt == 0.0:
|
|
return _fps.value
|
|
return _fps.update(1000.0 / dt)
|
|
|
|
|
|
def interpolate(y, new_length):
|
|
"""Intelligently resizes the array by linearly interpolating the values
|
|
|
|
Parameters
|
|
----------
|
|
y : np.array
|
|
Array that should be resized
|
|
|
|
new_length : int
|
|
The length of the new interpolated array
|
|
|
|
Returns
|
|
-------
|
|
z : np.array
|
|
New array with length of new_length that contains the interpolated
|
|
values of y.
|
|
"""
|
|
if len(y) == new_length:
|
|
return y
|
|
x_old = np.linspace(0, 1, len(y))
|
|
x_new = np.linspace(0, 1, new_length)
|
|
z = np.interp(x_new, x_old, y)
|
|
return z
|
|
|
|
|
|
r_filt = dsp.ExpFilter(np.tile(0.01, config.N_PIXELS // 2),
|
|
alpha_decay=0.2, alpha_rise=0.99)
|
|
g_filt = dsp.ExpFilter(np.tile(0.01, config.N_PIXELS // 2),
|
|
alpha_decay=0.05, alpha_rise=0.3)
|
|
b_filt = dsp.ExpFilter(np.tile(0.01, config.N_PIXELS // 2),
|
|
alpha_decay=0.1, alpha_rise=0.5)
|
|
common_mode = dsp.ExpFilter(np.tile(0.01, config.N_PIXELS // 2),
|
|
alpha_decay=0.99, alpha_rise=0.01)
|
|
p_filt = dsp.ExpFilter(np.tile(1, (3, config.N_PIXELS // 2)),
|
|
alpha_decay=0.1, alpha_rise=0.99)
|
|
p = np.tile(1.0, (3, config.N_PIXELS // 2))
|
|
gain = dsp.ExpFilter(np.tile(0.01, config.N_FFT_BINS),
|
|
alpha_decay=0.001, alpha_rise=0.99)
|
|
|
|
|
|
def visualize_scroll(y):
|
|
"""Effect that originates in the center and scrolls outwards"""
|
|
global p
|
|
y = np.copy(y)**2.0
|
|
gain.update(y)
|
|
y /= gain.value
|
|
y *= 255.0
|
|
r = int(max(y[:len(y) // 3]))
|
|
g = int(max(y[len(y) // 3: 2 * len(y) // 3]))
|
|
b = int(max(y[2 * len(y) // 3:]))
|
|
# Scrolling effect window
|
|
p = np.roll(p, 1, axis=1)
|
|
p *= 0.98
|
|
p = gaussian_filter1d(p, sigma=0.2)
|
|
# Create new color originating at the center
|
|
p[0, 0] = r
|
|
p[1, 0] = g
|
|
p[2, 0] = b
|
|
# Update the LED strip
|
|
return np.concatenate((p[:, ::-1], p), axis=1)
|
|
|
|
|
|
def visualize_energy(y):
|
|
"""Effect that expands from the center with increasing sound energy"""
|
|
global p
|
|
y = np.copy(y)
|
|
gain.update(y)
|
|
y /= gain.value
|
|
# Scale by the width of the LED strip
|
|
y *= float((config.N_PIXELS // 2) - 1)
|
|
# Map color channels according to energy in the different freq bands
|
|
scale = 0.9
|
|
r = int(np.mean(y[:len(y) // 3]**scale))
|
|
g = int(np.mean(y[len(y) // 3: 2 * len(y) // 3]**scale))
|
|
b = int(np.mean(y[2 * len(y) // 3:]**scale))
|
|
# Assign color to different frequency regions
|
|
p[0, :r] = 255.0
|
|
p[0, r:] = 0.0
|
|
p[1, :g] = 255.0
|
|
p[1, g:] = 0.0
|
|
p[2, :b] = 255.0
|
|
p[2, b:] = 0.0
|
|
p_filt.update(p)
|
|
p = np.round(p_filt.value)
|
|
# Apply substantial blur to smooth the edges
|
|
p[0, :] = gaussian_filter1d(p[0, :], sigma=4.0)
|
|
p[1, :] = gaussian_filter1d(p[1, :], sigma=4.0)
|
|
p[2, :] = gaussian_filter1d(p[2, :], sigma=4.0)
|
|
# Set the new pixel value
|
|
return np.concatenate((p[:, ::-1], p), axis=1)
|
|
|
|
prev_spectrum = np.tile(0.01, config.N_PIXELS // 2)
|
|
def visualize_spectrum(y):
|
|
"""Effect that maps the Mel filterbank frequencies onto the LED strip"""
|
|
global prev_spectrum
|
|
y = np.copy(interpolate(y, config.N_PIXELS // 2))
|
|
common_mode.update(gaussian_filter1d(y, sigma=2.0))
|
|
diff = y - prev_spectrum
|
|
prev_spectrum = np.copy(y)
|
|
r = gaussian_filter1d(y, sigma=0.5) - common_mode.value
|
|
# g = gaussian_filter1d(y, sigma=0.5) - common_mode.value
|
|
b = gaussian_filter1d(y, sigma=0.0) - common_mode.value
|
|
# Update temporal filters
|
|
r = r_filt.update(r)
|
|
# g = g_filt.update(g)
|
|
g = np.abs(diff)
|
|
b = b_filt.update(b)
|
|
# Mirror the color channels for symmetric output
|
|
pixel_r = np.concatenate((r[::-1], r))
|
|
pixel_g = np.concatenate((g[::-1], g))
|
|
pixel_b = np.concatenate((b[::-1], b))
|
|
output = np.array([pixel_r, pixel_g, pixel_b]) * 255.0
|
|
return output
|
|
|
|
|
|
fft_plot_filter = dsp.ExpFilter(np.tile(1e-1, config.N_FFT_BINS),
|
|
alpha_decay=0.5, alpha_rise=0.99)
|
|
mel_gain = dsp.ExpFilter(np.tile(1e-1, config.N_FFT_BINS),
|
|
alpha_decay=0.01, alpha_rise=0.99)
|
|
mel_smoothing = dsp.ExpFilter(np.tile(1e-1, config.N_FFT_BINS),
|
|
alpha_decay=0.5, alpha_rise=0.99)
|
|
volume = dsp.ExpFilter(config.MIN_VOLUME_THRESHOLD,
|
|
alpha_decay=0.02, alpha_rise=0.02)
|
|
|
|
# Keeps track of the number of buffer overflows
|
|
# Lots of buffer overflows could mean that FPS is set too high
|
|
buffer_overflows = 1
|
|
|
|
|
|
def microphone_update(stream):
|
|
global y_roll, prev_rms, prev_exp
|
|
# Retrieve and normalize the new audio samples
|
|
try:
|
|
y = np.fromstring(stream.read(samples_per_frame), dtype=np.int16)
|
|
except IOError:
|
|
y = y_roll[config.N_ROLLING_HISTORY - 1, :]
|
|
global buffer_overflows
|
|
print('Buffer overflows: {0}'.format(buffer_overflows))
|
|
buffer_overflows += 1
|
|
# Normalize samples between 0 and 1
|
|
y = y / 2.0**15
|
|
# Construct a rolling window of audio samples
|
|
y_roll = np.roll(y_roll, -1, axis=0)
|
|
y_roll[-1, :] = np.copy(y)
|
|
y_data = np.concatenate(y_roll, axis=0)
|
|
volume.update(np.nanmean(y_data ** 2))
|
|
|
|
if volume.value < config.MIN_VOLUME_THRESHOLD:
|
|
print('No audio input. Volume below threshold. Volume:', volume.value)
|
|
led.pixels = np.tile(0, (3, config.N_PIXELS))
|
|
led.update()
|
|
else:
|
|
# Transform audio input into the frequency domain
|
|
XS, YS = dsp.fft(y_data, window=np.hamming)
|
|
# Remove half of the FFT data because of symmetry
|
|
YS = YS[:len(YS) // 2]
|
|
XS = XS[:len(XS) // 2]
|
|
# Construct a Mel filterbank from the FFT data
|
|
mel = np.atleast_2d(np.abs(YS)).T * dsp.mel_y.T
|
|
# Scale data to values more suitable for visualization
|
|
mel = np.mean(mel, axis=0)
|
|
mel = mel**2.0
|
|
# Gain normalization
|
|
mel_gain.update(np.max(gaussian_filter1d(mel, sigma=1.0)))
|
|
mel = mel / mel_gain.value
|
|
mel = mel_smoothing.update(mel)
|
|
# Map filterbank output onto LED strip
|
|
output = visualization_effect(mel)
|
|
led.pixels = output
|
|
led.update()
|
|
|
|
# Plot filterbank output
|
|
x = np.linspace(config.MIN_FREQUENCY, config.MAX_FREQUENCY, len(mel))
|
|
mel_curve.setData(x=x, y=fft_plot_filter.update(mel))
|
|
# Plot the color channels
|
|
r_curve.setData(y=led.pixels[0])
|
|
g_curve.setData(y=led.pixels[1])
|
|
b_curve.setData(y=led.pixels[2])
|
|
if config.USE_GUI:
|
|
app.processEvents()
|
|
if config.DISPLAY_FPS:
|
|
print('FPS {:.0f} / {:.0f}'.format(frames_per_second(), config.FPS))
|
|
|
|
|
|
# Number of audio samples to read every time frame
|
|
samples_per_frame = int(config.MIC_RATE / config.FPS)
|
|
|
|
# Array containing the rolling audio sample window
|
|
y_roll = np.random.rand(config.N_ROLLING_HISTORY, samples_per_frame) / 1e16
|
|
|
|
visualization_effect = visualize_spectrum
|
|
"""Visualization effect to display on the LED strip"""
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if config.USE_GUI:
|
|
import pyqtgraph as pg
|
|
from pyqtgraph.Qt import QtGui, QtCore
|
|
# Create GUI window
|
|
app = QtGui.QApplication([])
|
|
view = pg.GraphicsView()
|
|
layout = pg.GraphicsLayout(border=(100,100,100))
|
|
view.setCentralItem(layout)
|
|
view.show()
|
|
view.setWindowTitle('Visualization')
|
|
view.resize(800,600)
|
|
# Mel filterbank plot
|
|
fft_plot = layout.addPlot(title='Filterbank Output', colspan=3)
|
|
fft_plot.setRange(yRange=[-0.1, 1.2])
|
|
fft_plot.disableAutoRange(axis=pg.ViewBox.YAxis)
|
|
x_data = np.array(range(1, config.N_FFT_BINS + 1))
|
|
mel_curve = pg.PlotCurveItem()
|
|
mel_curve.setData(x=x_data, y=x_data*0)
|
|
fft_plot.addItem(mel_curve)
|
|
# Visualization plot
|
|
layout.nextRow()
|
|
led_plot = layout.addPlot(title='Visualization Output', colspan=3)
|
|
led_plot.setRange(yRange=[-5, 260])
|
|
led_plot.disableAutoRange(axis=pg.ViewBox.YAxis)
|
|
# Pen for each of the color channel curves
|
|
r_pen = pg.mkPen((255, 30, 30, 200), width=4)
|
|
g_pen = pg.mkPen((30, 255, 30, 200), width=4)
|
|
b_pen = pg.mkPen((30, 30, 255, 200), width=4)
|
|
# Color channel curves
|
|
r_curve = pg.PlotCurveItem(pen=r_pen)
|
|
g_curve = pg.PlotCurveItem(pen=g_pen)
|
|
b_curve = pg.PlotCurveItem(pen=b_pen)
|
|
# Define x data
|
|
x_data = np.array(range(1, config.N_PIXELS + 1))
|
|
r_curve.setData(x=x_data, y=x_data*0)
|
|
g_curve.setData(x=x_data, y=x_data*0)
|
|
b_curve.setData(x=x_data, y=x_data*0)
|
|
# Add curves to plot
|
|
led_plot.addItem(r_curve)
|
|
led_plot.addItem(g_curve)
|
|
led_plot.addItem(b_curve)
|
|
# Frequency range label
|
|
freq_label = pg.LabelItem('')
|
|
# Frequency slider
|
|
def freq_slider_change(tick):
|
|
minf = freq_slider.tickValue(0)**2.0 * (config.MIC_RATE / 2.0)
|
|
maxf = freq_slider.tickValue(1)**2.0 * (config.MIC_RATE / 2.0)
|
|
t = 'Frequency range: {:.0f} - {:.0f} Hz'.format(minf, maxf)
|
|
freq_label.setText(t)
|
|
config.MIN_FREQUENCY = minf
|
|
config.MAX_FREQUENCY = maxf
|
|
dsp.create_mel_bank()
|
|
freq_slider = pg.TickSliderItem(orientation='bottom', allowAdd=False)
|
|
freq_slider.addTick((config.MIN_FREQUENCY / (config.MIC_RATE / 2.0))**0.5)
|
|
freq_slider.addTick((config.MAX_FREQUENCY / (config.MIC_RATE / 2.0))**0.5)
|
|
freq_slider.tickMoveFinished = freq_slider_change
|
|
freq_label.setText('Frequency range: {} - {} Hz'.format(
|
|
config.MIN_FREQUENCY,
|
|
config.MAX_FREQUENCY))
|
|
# Effect selection
|
|
active_color = '#16dbeb'
|
|
inactive_color = '#FFFFFF'
|
|
def energy_click(x):
|
|
global visualization_effect
|
|
visualization_effect = visualize_energy
|
|
energy_label.setText('Energy', color=active_color)
|
|
scroll_label.setText('Scroll', color=inactive_color)
|
|
spectrum_label.setText('Spectrum', color=inactive_color)
|
|
def scroll_click(x):
|
|
global visualization_effect
|
|
visualization_effect = visualize_scroll
|
|
energy_label.setText('Energy', color=inactive_color)
|
|
scroll_label.setText('Scroll', color=active_color)
|
|
spectrum_label.setText('Spectrum', color=inactive_color)
|
|
def spectrum_click(x):
|
|
global visualization_effect
|
|
visualization_effect = visualize_spectrum
|
|
energy_label.setText('Energy', color=inactive_color)
|
|
scroll_label.setText('Scroll', color=inactive_color)
|
|
spectrum_label.setText('Spectrum', color=active_color)
|
|
# Create effect "buttons" (labels with click event)
|
|
energy_label = pg.LabelItem('Energy')
|
|
scroll_label = pg.LabelItem('Scroll')
|
|
spectrum_label = pg.LabelItem('Spectrum')
|
|
energy_label.mousePressEvent = energy_click
|
|
scroll_label.mousePressEvent = scroll_click
|
|
spectrum_label.mousePressEvent = spectrum_click
|
|
energy_click(0)
|
|
# Layout
|
|
layout.nextRow()
|
|
layout.addItem(freq_label, colspan=3)
|
|
layout.nextRow()
|
|
layout.addItem(freq_slider, colspan=3)
|
|
layout.nextRow()
|
|
layout.addItem(energy_label)
|
|
layout.addItem(scroll_label)
|
|
layout.addItem(spectrum_label)
|
|
# Initialize LEDs
|
|
led.update()
|
|
# Start listening to live audio stream
|
|
microphone.start_stream(microphone_update)
|