[android/platform-hardware-interfaces.git] / neuralnetworks / 1.2 / vts / functional / ValidateModel.cpp
1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
17 #define LOG_TAG "neuralnetworks_hidl_hal_test"
19 #include "VtsHalNeuralnetworks.h"
21 #include "Callbacks.h"
23 namespace android {
24 namespace hardware {
25 namespace neuralnetworks {
26 namespace V1_2 {
28 using V1_0::IPreparedModel;
29 using V1_0::OperandLifeTime;
30 using V1_1::ExecutionPreference;
32 namespace vts {
33 namespace functional {
35 using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
36 using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
38 ///////////////////////// UTILITY FUNCTIONS /////////////////////////
40 static void validateGetSupportedOperations(const sp<IDevice>& device, const std::string& message,
41 const Model& model) {
42 SCOPED_TRACE(message + " [getSupportedOperations_1_2]");
44 Return<void> ret =
45 device->getSupportedOperations_1_2(model, [&](ErrorStatus status, const hidl_vec<bool>&) {
46 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
47 });
48 EXPECT_TRUE(ret.isOk());
49 }
51 static void validatePrepareModel(const sp<IDevice>& device, const std::string& message,
52 const Model& model, ExecutionPreference preference) {
53 SCOPED_TRACE(message + " [prepareModel_1_2]");
55 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
56 ASSERT_NE(nullptr, preparedModelCallback.get());
57 Return<ErrorStatus> prepareLaunchStatus =
58 device->prepareModel_1_2(model, preference, preparedModelCallback);
59 ASSERT_TRUE(prepareLaunchStatus.isOk());
60 ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
62 preparedModelCallback->wait();
63 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
64 ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
65 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
66 ASSERT_EQ(nullptr, preparedModel.get());
67 }
69 static bool validExecutionPreference(ExecutionPreference preference) {
70 return preference == ExecutionPreference::LOW_POWER ||
71 preference == ExecutionPreference::FAST_SINGLE_ANSWER ||
72 preference == ExecutionPreference::SUSTAINED_SPEED;
73 }
75 // Primary validation function. This function will take a valid model, apply a
76 // mutation to it to invalidate the model, then pass it to interface calls that
77 // use the model. Note that the model here is passed by value, and any mutation
78 // to the model does not leave this function.
79 static void validate(const sp<IDevice>& device, const std::string& message, Model model,
80 const std::function<void(Model*)>& mutation,
81 ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER) {
82 mutation(&model);
83 if (validExecutionPreference(preference)) {
84 validateGetSupportedOperations(device, message, model);
85 }
86 validatePrepareModel(device, message, model, preference);
87 }
89 // Delete element from hidl_vec. hidl_vec doesn't support a "remove" operation,
90 // so this is efficiently accomplished by moving the element to the end and
91 // resizing the hidl_vec to one less.
92 template <typename Type>
93 static void hidl_vec_removeAt(hidl_vec<Type>* vec, uint32_t index) {
94 if (vec) {
95 std::rotate(vec->begin() + index, vec->begin() + index + 1, vec->end());
96 vec->resize(vec->size() - 1);
97 }
98 }
100 template <typename Type>
101 static uint32_t hidl_vec_push_back(hidl_vec<Type>* vec, const Type& value) {
102 // assume vec is valid
103 const uint32_t index = vec->size();
104 vec->resize(index + 1);
105 (*vec)[index] = value;
106 return index;
107 }
109 static uint32_t addOperand(Model* model) {
110 return hidl_vec_push_back(&model->operands,
111 {
112 .type = OperandType::INT32,
113 .dimensions = {},
114 .numberOfConsumers = 0,
115 .scale = 0.0f,
116 .zeroPoint = 0,
117 .lifetime = OperandLifeTime::MODEL_INPUT,
118 .location = {.poolIndex = 0, .offset = 0, .length = 0},
119 });
120 }
122 static uint32_t addOperand(Model* model, OperandLifeTime lifetime) {
123 uint32_t index = addOperand(model);
124 model->operands[index].numberOfConsumers = 1;
125 model->operands[index].lifetime = lifetime;
126 return index;
127 }
129 ///////////////////////// VALIDATE MODEL OPERAND TYPE /////////////////////////
131 static const int32_t invalidOperandTypes[] = {
132 static_cast<int32_t>(OperandType::FLOAT32) - 1, // lower bound fundamental
133 static_cast<int32_t>(OperandType::TENSOR_QUANT16_ASYMM) + 1, // upper bound fundamental
134 static_cast<int32_t>(OperandType::OEM) - 1, // lower bound OEM
135 static_cast<int32_t>(OperandType::TENSOR_OEM_BYTE) + 1, // upper bound OEM
136 };
138 static void mutateOperandTypeTest(const sp<IDevice>& device, const Model& model) {
139 for (size_t operand = 0; operand < model.operands.size(); ++operand) {
140 for (int32_t invalidOperandType : invalidOperandTypes) {
141 const std::string message = "mutateOperandTypeTest: operand " +
142 std::to_string(operand) + " set to value " +
143 std::to_string(invalidOperandType);
144 validate(device, message, model, [operand, invalidOperandType](Model* model) {
145 model->operands[operand].type = static_cast<OperandType>(invalidOperandType);
146 });
147 }
148 }
149 }
151 ///////////////////////// VALIDATE OPERAND RANK /////////////////////////
153 static uint32_t getInvalidRank(OperandType type) {
154 switch (type) {
155 case OperandType::FLOAT32:
156 case OperandType::INT32:
157 case OperandType::UINT32:
158 case OperandType::BOOL:
159 return 1;
160 case OperandType::TENSOR_FLOAT16:
161 case OperandType::TENSOR_FLOAT32:
162 case OperandType::TENSOR_INT32:
163 case OperandType::TENSOR_QUANT8_ASYMM:
164 case OperandType::TENSOR_QUANT16_ASYMM:
165 return 0;
166 default:
167 return 0;
168 }
169 }
171 static void mutateOperandRankTest(const sp<IDevice>& device, const Model& model) {
172 for (size_t operand = 0; operand < model.operands.size(); ++operand) {
173 const uint32_t invalidRank = getInvalidRank(model.operands[operand].type);
174 const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) +
175 " has rank of " + std::to_string(invalidRank);
176 validate(device, message, model, [operand, invalidRank](Model* model) {
177 model->operands[operand].dimensions = std::vector<uint32_t>(invalidRank, 0);
178 });
179 }
180 }
182 ///////////////////////// VALIDATE OPERAND SCALE /////////////////////////
184 static float getInvalidScale(OperandType type) {
185 switch (type) {
186 case OperandType::FLOAT32:
187 case OperandType::INT32:
188 case OperandType::UINT32:
189 case OperandType::BOOL:
190 case OperandType::TENSOR_FLOAT16:
191 case OperandType::TENSOR_FLOAT32:
192 return 1.0f;
193 case OperandType::TENSOR_INT32:
194 return -1.0f;
195 case OperandType::TENSOR_QUANT8_ASYMM:
196 case OperandType::TENSOR_QUANT16_ASYMM:
197 return 0.0f;
198 default:
199 return 0.0f;
200 }
201 }
203 static void mutateOperandScaleTest(const sp<IDevice>& device, const Model& model) {
204 for (size_t operand = 0; operand < model.operands.size(); ++operand) {
205 const float invalidScale = getInvalidScale(model.operands[operand].type);
206 const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) +
207 " has scale of " + std::to_string(invalidScale);
208 validate(device, message, model, [operand, invalidScale](Model* model) {
209 model->operands[operand].scale = invalidScale;
210 });
211 }
212 }
214 ///////////////////////// VALIDATE OPERAND ZERO POINT /////////////////////////
216 static std::vector<int32_t> getInvalidZeroPoints(OperandType type) {
217 switch (type) {
218 case OperandType::FLOAT32:
219 case OperandType::INT32:
220 case OperandType::UINT32:
221 case OperandType::BOOL:
222 case OperandType::TENSOR_FLOAT16:
223 case OperandType::TENSOR_FLOAT32:
224 case OperandType::TENSOR_INT32:
225 return {1};
226 case OperandType::TENSOR_QUANT8_ASYMM:
227 case OperandType::TENSOR_QUANT16_ASYMM:
228 return {-1, 256};
229 default:
230 return {};
231 }
232 }
234 static void mutateOperandZeroPointTest(const sp<IDevice>& device, const Model& model) {
235 for (size_t operand = 0; operand < model.operands.size(); ++operand) {
236 const std::vector<int32_t> invalidZeroPoints =
237 getInvalidZeroPoints(model.operands[operand].type);
238 for (int32_t invalidZeroPoint : invalidZeroPoints) {
239 const std::string message = "mutateOperandZeroPointTest: operand " +
240 std::to_string(operand) + " has zero point of " +
241 std::to_string(invalidZeroPoint);
242 validate(device, message, model, [operand, invalidZeroPoint](Model* model) {
243 model->operands[operand].zeroPoint = invalidZeroPoint;
244 });
245 }
246 }
247 }
249 ///////////////////////// VALIDATE EXTRA ??? /////////////////////////
251 // TODO: Operand::lifetime
252 // TODO: Operand::location
254 ///////////////////////// VALIDATE OPERATION OPERAND TYPE /////////////////////////
256 static void mutateOperand(Operand* operand, OperandType type) {
257 Operand newOperand = *operand;
258 newOperand.type = type;
259 switch (type) {
260 case OperandType::FLOAT32:
261 case OperandType::INT32:
262 case OperandType::UINT32:
263 case OperandType::BOOL:
264 newOperand.dimensions = hidl_vec<uint32_t>();
265 newOperand.scale = 0.0f;
266 newOperand.zeroPoint = 0;
267 break;
268 case OperandType::TENSOR_FLOAT16:
269 case OperandType::TENSOR_FLOAT32:
270 newOperand.dimensions =
271 operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
272 newOperand.scale = 0.0f;
273 newOperand.zeroPoint = 0;
274 break;
275 case OperandType::TENSOR_INT32:
276 newOperand.dimensions =
277 operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
278 newOperand.zeroPoint = 0;
279 break;
280 case OperandType::TENSOR_QUANT8_ASYMM:
281 case OperandType::TENSOR_QUANT16_ASYMM:
282 newOperand.dimensions =
283 operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
284 newOperand.scale = operand->scale != 0.0f ? operand->scale : 1.0f;
285 break;
286 case OperandType::OEM:
287 case OperandType::TENSOR_OEM_BYTE:
288 default:
289 break;
290 }
291 *operand = newOperand;
292 }
294 static bool mutateOperationOperandTypeSkip(size_t operand, const Model& model) {
295 // LSH_PROJECTION's second argument is allowed to have any type. This is the
296 // only operation that currently has a type that can be anything independent
297 // from any other type. Changing the operand type to any other type will
298 // result in a valid model for LSH_PROJECTION. If this is the case, skip the
299 // test.
300 for (const Operation& operation : model.operations) {
301 if (operation.type == OperationType::LSH_PROJECTION && operand == operation.inputs[1]) {
302 return true;
303 }
304 }
305 return false;
306 }
308 static void mutateOperationOperandTypeTest(const sp<IDevice>& device, const Model& model) {
309 for (size_t operand = 0; operand < model.operands.size(); ++operand) {
310 if (mutateOperationOperandTypeSkip(operand, model)) {
311 continue;
312 }
313 for (OperandType invalidOperandType : hidl_enum_range<OperandType>{}) {
314 // Do not test OEM types
315 if (invalidOperandType == model.operands[operand].type ||
316 invalidOperandType == OperandType::OEM ||
317 invalidOperandType == OperandType::TENSOR_OEM_BYTE) {
318 continue;
319 }
320 const std::string message = "mutateOperationOperandTypeTest: operand " +
321 std::to_string(operand) + " set to type " +
322 toString(invalidOperandType);
323 validate(device, message, model, [operand, invalidOperandType](Model* model) {
324 mutateOperand(&model->operands[operand], invalidOperandType);
325 });
326 }
327 }
328 }
330 ///////////////////////// VALIDATE MODEL OPERATION TYPE /////////////////////////
332 static const int32_t invalidOperationTypes[] = {
333 static_cast<int32_t>(OperationType::ADD) - 1, // lower bound fundamental
334 static_cast<int32_t>(OperationType::TRANSPOSE) + 1, // upper bound fundamental
335 static_cast<int32_t>(OperationType::OEM_OPERATION) - 1, // lower bound OEM
336 static_cast<int32_t>(OperationType::OEM_OPERATION) + 1, // upper bound OEM
337 };
339 static void mutateOperationTypeTest(const sp<IDevice>& device, const Model& model) {
340 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
341 for (int32_t invalidOperationType : invalidOperationTypes) {
342 const std::string message = "mutateOperationTypeTest: operation " +
343 std::to_string(operation) + " set to value " +
344 std::to_string(invalidOperationType);
345 validate(device, message, model, [operation, invalidOperationType](Model* model) {
346 model->operations[operation].type =
347 static_cast<OperationType>(invalidOperationType);
348 });
349 }
350 }
351 }
353 ///////////////////////// VALIDATE MODEL OPERATION INPUT OPERAND INDEX /////////////////////////
355 static void mutateOperationInputOperandIndexTest(const sp<IDevice>& device, const Model& model) {
356 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
357 const uint32_t invalidOperand = model.operands.size();
358 for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) {
359 const std::string message = "mutateOperationInputOperandIndexTest: operation " +
360 std::to_string(operation) + " input " +
361 std::to_string(input);
362 validate(device, message, model, [operation, input, invalidOperand](Model* model) {
363 model->operations[operation].inputs[input] = invalidOperand;
364 });
365 }
366 }
367 }
369 ///////////////////////// VALIDATE MODEL OPERATION OUTPUT OPERAND INDEX /////////////////////////
371 static void mutateOperationOutputOperandIndexTest(const sp<IDevice>& device, const Model& model) {
372 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
373 const uint32_t invalidOperand = model.operands.size();
374 for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) {
375 const std::string message = "mutateOperationOutputOperandIndexTest: operation " +
376 std::to_string(operation) + " output " +
377 std::to_string(output);
378 validate(device, message, model, [operation, output, invalidOperand](Model* model) {
379 model->operations[operation].outputs[output] = invalidOperand;
380 });
381 }
382 }
383 }
385 ///////////////////////// REMOVE OPERAND FROM EVERYTHING /////////////////////////
387 static void removeValueAndDecrementGreaterValues(hidl_vec<uint32_t>* vec, uint32_t value) {
388 if (vec) {
389 // remove elements matching "value"
390 auto last = std::remove(vec->begin(), vec->end(), value);
391 vec->resize(std::distance(vec->begin(), last));
393 // decrement elements exceeding "value"
394 std::transform(vec->begin(), vec->end(), vec->begin(),
395 [value](uint32_t v) { return v > value ? v-- : v; });
396 }
397 }
399 static void removeOperand(Model* model, uint32_t index) {
400 hidl_vec_removeAt(&model->operands, index);
401 for (Operation& operation : model->operations) {
402 removeValueAndDecrementGreaterValues(&operation.inputs, index);
403 removeValueAndDecrementGreaterValues(&operation.outputs, index);
404 }
405 removeValueAndDecrementGreaterValues(&model->inputIndexes, index);
406 removeValueAndDecrementGreaterValues(&model->outputIndexes, index);
407 }
409 static void removeOperandTest(const sp<IDevice>& device, const Model& model) {
410 for (size_t operand = 0; operand < model.operands.size(); ++operand) {
411 const std::string message = "removeOperandTest: operand " + std::to_string(operand);
412 validate(device, message, model,
413 [operand](Model* model) { removeOperand(model, operand); });
414 }
415 }
417 ///////////////////////// REMOVE OPERATION /////////////////////////
419 static void removeOperation(Model* model, uint32_t index) {
420 for (uint32_t operand : model->operations[index].inputs) {
421 model->operands[operand].numberOfConsumers--;
422 }
423 hidl_vec_removeAt(&model->operations, index);
424 }
426 static void removeOperationTest(const sp<IDevice>& device, const Model& model) {
427 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
428 const std::string message = "removeOperationTest: operation " + std::to_string(operation);
429 validate(device, message, model,
430 [operation](Model* model) { removeOperation(model, operation); });
431 }
432 }
434 ///////////////////////// REMOVE OPERATION INPUT /////////////////////////
436 static void removeOperationInputTest(const sp<IDevice>& device, const Model& model) {
437 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
438 for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) {
439 const Operation& op = model.operations[operation];
440 // CONCATENATION has at least 2 inputs, with the last element being
441 // INT32. Skip this test if removing one of CONCATENATION's
442 // inputs still produces a valid model.
443 if (op.type == OperationType::CONCATENATION && op.inputs.size() > 2 &&
444 input != op.inputs.size() - 1) {
445 continue;
446 }
447 const std::string message = "removeOperationInputTest: operation " +
448 std::to_string(operation) + ", input " +
449 std::to_string(input);
450 validate(device, message, model, [operation, input](Model* model) {
451 uint32_t operand = model->operations[operation].inputs[input];
452 model->operands[operand].numberOfConsumers--;
453 hidl_vec_removeAt(&model->operations[operation].inputs, input);
454 });
455 }
456 }
457 }
459 ///////////////////////// REMOVE OPERATION OUTPUT /////////////////////////
461 static void removeOperationOutputTest(const sp<IDevice>& device, const Model& model) {
462 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
463 for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) {
464 const std::string message = "removeOperationOutputTest: operation " +
465 std::to_string(operation) + ", output " +
466 std::to_string(output);
467 validate(device, message, model, [operation, output](Model* model) {
468 hidl_vec_removeAt(&model->operations[operation].outputs, output);
469 });
470 }
471 }
472 }
474 ///////////////////////// MODEL VALIDATION /////////////////////////
476 // TODO: remove model input
477 // TODO: remove model output
478 // TODO: add unused operation
480 ///////////////////////// ADD OPERATION INPUT /////////////////////////
482 static void addOperationInputTest(const sp<IDevice>& device, const Model& model) {
483 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
484 const std::string message = "addOperationInputTest: operation " + std::to_string(operation);
485 validate(device, message, model, [operation](Model* model) {
486 uint32_t index = addOperand(model, OperandLifeTime::MODEL_INPUT);
487 hidl_vec_push_back(&model->operations[operation].inputs, index);
488 hidl_vec_push_back(&model->inputIndexes, index);
489 });
490 }
491 }
493 ///////////////////////// ADD OPERATION OUTPUT /////////////////////////
495 static void addOperationOutputTest(const sp<IDevice>& device, const Model& model) {
496 for (size_t operation = 0; operation < model.operations.size(); ++operation) {
497 const std::string message =
498 "addOperationOutputTest: operation " + std::to_string(operation);
499 validate(device, message, model, [operation](Model* model) {
500 uint32_t index = addOperand(model, OperandLifeTime::MODEL_OUTPUT);
501 hidl_vec_push_back(&model->operations[operation].outputs, index);
502 hidl_vec_push_back(&model->outputIndexes, index);
503 });
504 }
505 }
507 ///////////////////////// VALIDATE EXECUTION PREFERENCE /////////////////////////
509 static const int32_t invalidExecutionPreferences[] = {
510 static_cast<int32_t>(ExecutionPreference::LOW_POWER) - 1, // lower bound
511 static_cast<int32_t>(ExecutionPreference::SUSTAINED_SPEED) + 1, // upper bound
512 };
514 static void mutateExecutionPreferenceTest(const sp<IDevice>& device, const Model& model) {
515 for (int32_t preference : invalidExecutionPreferences) {
516 const std::string message =
517 "mutateExecutionPreferenceTest: preference " + std::to_string(preference);
518 validate(device, message, model, [](Model*) {},
519 static_cast<ExecutionPreference>(preference));
520 }
521 }
523 ////////////////////////// ENTRY POINT //////////////////////////////
525 void ValidationTest::validateModel(const Model& model) {
526 mutateOperandTypeTest(device, model);
527 mutateOperandRankTest(device, model);
528 mutateOperandScaleTest(device, model);
529 mutateOperandZeroPointTest(device, model);
530 mutateOperationOperandTypeTest(device, model);
531 mutateOperationTypeTest(device, model);
532 mutateOperationInputOperandIndexTest(device, model);
533 mutateOperationOutputOperandIndexTest(device, model);
534 removeOperandTest(device, model);
535 removeOperationTest(device, model);
536 removeOperationInputTest(device, model);
537 removeOperationOutputTest(device, model);
538 addOperationInputTest(device, model);
539 addOperationOutputTest(device, model);
540 mutateExecutionPreferenceTest(device, model);
541 }
543 } // namespace functional
544 } // namespace vts
545 } // namespace V1_2
546 } // namespace neuralnetworks
547 } // namespace hardware
548 } // namespace android