Skip to content

Commit b4b7708

Browse files
plancherb1plancherb1
committed
adding micro speech with modifications
1 parent e08dca5 commit b4b7708

23 files changed

+3703
-0
lines changed

‎examples/micro_speech/arduino_audio_provider.cpp

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
17+
18+
Licensed under the Apache License, Version 2.0 (the "License");
19+
you may not use this file except in compliance with the License.
20+
You may obtain a copy of the License at
21+
22+
http://www.apache.org/licenses/LICENSE-2.0
23+
24+
Unless required by applicable law or agreed to in writing, software
25+
distributed under the License is distributed on an "AS IS" BASIS,
26+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
27+
See the License for the specific language governing permissions and
28+
limitations under the License.
29+
==============================================================================*/
30+
31+
#if defined(ARDUINO) && !defined(ARDUINO_ARDUINO_NANO33BLE)
32+
#define ARDUINO_EXCLUDE_CODE
33+
#endif // defined(ARDUINO) && !defined(ARDUINO_ARDUINO_NANO33BLE)
34+
35+
#ifndef ARDUINO_EXCLUDE_CODE
36+
37+
#include "audio_provider.h"
38+
39+
#include "PDM.h"
40+
#include "micro_features_micro_model_settings.h"
41+
42+
namespace {
43+
bool g_is_audio_initialized = false;
44+
// An internal buffer able to fit 16x our sample size
45+
constexpr int kAudioCaptureBufferSize = DEFAULT_PDM_BUFFER_SIZE * 16;
46+
int16_t g_audio_capture_buffer[kAudioCaptureBufferSize];
47+
// A buffer that holds our output
48+
int16_t g_audio_output_buffer[kMaxAudioSampleSize];
49+
// Mark as volatile so we can check in a while loop to see if
50+
// any samples have arrived yet.
51+
volatile int32_t g_latest_audio_timestamp = 0;
52+
} // namespace
53+
54+
void CaptureSamples() {
55+
// This is how many bytes of new data we have each time this is called
56+
const int number_of_samples = DEFAULT_PDM_BUFFER_SIZE/2;
57+
// Calculate what timestamp the last audio sample represents
58+
const int32_t time_in_ms =
59+
g_latest_audio_timestamp +
60+
(number_of_samples / (kAudioSampleFrequency / 1000));
61+
// Determine the index, in the history of all samples, of the last sample
62+
const int32_t start_sample_offset =
63+
g_latest_audio_timestamp * (kAudioSampleFrequency / 1000);
64+
// Determine the index of this sample in our ring buffer
65+
const int capture_index = start_sample_offset % kAudioCaptureBufferSize;
66+
// Read the data to the correct place in our buffer
67+
PDM.read(g_audio_capture_buffer + capture_index, DEFAULT_PDM_BUFFER_SIZE);
68+
// This is how we let the outside world know that new audio data has arrived.
69+
g_latest_audio_timestamp = time_in_ms;
70+
}
71+
72+
TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) {
73+
// Hook up the callback that will be called with each sample
74+
PDM.onReceive(CaptureSamples);
75+
// Start listening for audio: MONO @ 16KHz with gain at 20
76+
PDM.begin(1, kAudioSampleFrequency);
77+
PDM.setGain(20);
78+
// Block until we have our first audio sample
79+
while (!g_latest_audio_timestamp) {
80+
}
81+
82+
return kTfLiteOk;
83+
}
84+
85+
TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
86+
int start_ms, int duration_ms,
87+
int* audio_samples_size, int16_t** audio_samples) {
88+
// Set everything up to start receiving audio
89+
if (!g_is_audio_initialized) {
90+
TfLiteStatus init_status = InitAudioRecording(error_reporter);
91+
if (init_status != kTfLiteOk) {
92+
return init_status;
93+
}
94+
g_is_audio_initialized = true;
95+
}
96+
// This next part should only be called when the main thread notices that the
97+
// latest audio sample data timestamp has changed, so that there's new data
98+
// in the capture ring buffer. The ring buffer will eventually wrap around and
99+
// overwrite the data, but the assumption is that the main thread is checking
100+
// often enough and the buffer is large enough that this call will be made
101+
// before that happens.
102+
103+
// Determine the index, in the history of all samples, of the first
104+
// sample we want
105+
const int start_offset = start_ms * (kAudioSampleFrequency / 1000);
106+
// Determine how many samples we want in total
107+
const int duration_sample_count =
108+
duration_ms * (kAudioSampleFrequency / 1000);
109+
for (int i = 0; i < duration_sample_count; ++i) {
110+
// For each sample, transform its index in the history of all samples into
111+
// its index in g_audio_capture_buffer
112+
const int capture_index = (start_offset + i) % kAudioCaptureBufferSize;
113+
// Write the sample to the output buffer
114+
g_audio_output_buffer[i] = g_audio_capture_buffer[capture_index];
115+
}
116+
117+
// Set pointers to provide access to the audio
118+
*audio_samples_size = kMaxAudioSampleSize;
119+
*audio_samples = g_audio_output_buffer;
120+
121+
return kTfLiteOk;
122+
}
123+
124+
int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }
125+
126+
#endif // ARDUINO_EXCLUDE_CODE

‎examples/micro_speech/arduino_command_responder.cpp

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
#if defined(ARDUINO) && !defined(ARDUINO_ARDUINO_NANO33BLE)
17+
#define ARDUINO_EXCLUDE_CODE
18+
#endif // defined(ARDUINO) && !defined(ARDUINO_ARDUINO_NANO33BLE)
19+
20+
#ifndef ARDUINO_EXCLUDE_CODE
21+
22+
#include "command_responder.h"
23+
24+
#include "Arduino.h"
25+
26+
// Toggles the built-in LED every inference, and lights a colored LED depending
27+
// on which word was detected.
28+
void RespondToCommand(tflite::ErrorReporter* error_reporter,
29+
int32_t current_time, const char* found_command,
30+
uint8_t score, bool is_new_command) {
31+
static bool is_initialized = false;
32+
if (!is_initialized) {
33+
pinMode(LED_BUILTIN, OUTPUT);
34+
// Pins for the built-in RGB LEDs on the Arduino Nano 33 BLE Sense
35+
pinMode(LEDR, OUTPUT);
36+
pinMode(LEDG, OUTPUT);
37+
pinMode(LEDB, OUTPUT);
38+
// Ensure the LED is off by default.
39+
// Note: The RGB LEDs on the Arduino Nano 33 BLE
40+
// Sense are on when the pin is LOW, off when HIGH.
41+
digitalWrite(LEDR, HIGH);
42+
digitalWrite(LEDG, HIGH);
43+
digitalWrite(LEDB, HIGH);
44+
is_initialized = true;
45+
}
46+
static int32_t last_command_time = 0;
47+
static int count = 0;
48+
static int certainty = 220;
49+
50+
if (is_new_command) {
51+
TF_LITE_REPORT_ERROR(error_reporter, "Heard %s (%d) @%dms", found_command,
52+
score, current_time);
53+
// If we hear a command, light up the appropriate LED
54+
if (found_command[0] == 'y') {
55+
last_command_time = current_time;
56+
digitalWrite(LEDG, LOW); // Green for yes
57+
}
58+
59+
if (found_command[0] == 'n') {
60+
last_command_time = current_time;
61+
digitalWrite(LEDR, LOW); // Red for no
62+
}
63+
64+
if (found_command[0] == 'u') {
65+
last_command_time = current_time;
66+
digitalWrite(LEDB, LOW); // Blue for unknown
67+
}
68+
}
69+
70+
// If last_command_time is non-zero but was >3 seconds ago, zero it
71+
// and switch off the LED.
72+
if (last_command_time != 0) {
73+
if (last_command_time < (current_time - 3000)) {
74+
last_command_time = 0;
75+
digitalWrite(LED_BUILTIN, LOW);
76+
digitalWrite(LEDR, HIGH);
77+
digitalWrite(LEDG, HIGH);
78+
digitalWrite(LEDB, HIGH);
79+
}
80+
// If it is non-zero but <3 seconds ago, do nothing.
81+
return;
82+
}
83+
84+
// Otherwise, toggle the LED every time an inference is performed.
85+
++count;
86+
if (count & 1) {
87+
digitalWrite(LED_BUILTIN, HIGH);
88+
} else {
89+
digitalWrite(LED_BUILTIN, LOW);
90+
}
91+
}
92+
93+
#endif // ARDUINO_EXCLUDE_CODE

‎examples/micro_speech/arduino_main.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
#include "main_functions.h"
17+
18+
// Arduino automatically calls the setup() and loop() functions in a sketch, so
19+
// where other systems need their own main routine in this file, it can be left
20+
// empty.

‎examples/micro_speech/audio_provider.h

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
17+
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
18+
19+
#include "tensorflow/lite/c/common.h"
20+
#include "tensorflow/lite/micro/micro_error_reporter.h"
21+
22+
// This is an abstraction around an audio source like a microphone, and is
23+
// expected to return 16-bit PCM sample data for a given point in time. The
24+
// sample data itself should be used as quickly as possible by the caller, since
25+
// to allow memory optimizations there are no guarantees that the samples won't
26+
// be overwritten by new data in the future. In practice, implementations should
27+
// ensure that there's a reasonable time allowed for clients to access the data
28+
// before any reuse.
29+
// The reference implementation can have no platform-specific dependencies, so
30+
// it just returns an array filled with zeros. For real applications, you should
31+
// ensure there's a specialized implementation that accesses hardware APIs.
32+
TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
33+
int start_ms, int duration_ms,
34+
int* audio_samples_size, int16_t** audio_samples);
35+
36+
// Returns the time that audio data was last captured in milliseconds. There's
37+
// no contract about what time zero represents, the accuracy, or the granularity
38+
// of the result. Subsequent calls will generally not return a lower value, but
39+
// even that's not guaranteed if there's an overflow wraparound.
40+
// The reference implementation of this function just returns a constantly
41+
// incrementing value for each call, since it would need a non-portable platform
42+
// call to access time information. For real applications, you'll need to write
43+
// your own platform-specific implementation.
44+
int32_t LatestAudioTimestamp();
45+
46+
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_

‎examples/micro_speech/command_responder.h

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
// Provides an interface to take an action based on an audio command.
17+
18+
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_
19+
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_
20+
21+
#include "tensorflow/lite/c/common.h"
22+
#include "tensorflow/lite/micro/micro_error_reporter.h"
23+
24+
// Called every time the results of an audio recognition run are available. The
25+
// human-readable name of any recognized command is in the `found_command`
26+
// argument, `score` has the numerical confidence, and `is_new_command` is set
27+
// if the previous command was different to this one.
28+
void RespondToCommand(tflite::ErrorReporter* error_reporter,
29+
int32_t current_time, const char* found_command,
30+
uint8_t score, bool is_new_command);
31+
32+
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_

0 commit comments

Comments
 (0)