前言
机器学习套件是一个移动 SDK，将 Google 的设备端机器学习专业知识运用于 Android 和 iOS 应用。使用我们强大而易用的 Vision API 和 Natural Language API 解决应用中的常见挑战，或打造全新的用户体验。所有功能均由 Google 一流的机器学习模型提供支持，可免费使用。
学习指南：https://developers.google.cn/ml-kit/guides?hl=zh-cn
效果图
书写识别
准备工作
1. 在 app/build.gradle 添加如下依赖
implementation 'com.google.mlkit:digital-ink-recognition:18.0.0'
2. 创建 StrokeManager.java
/**
* author: Kevin-Dev
* date: 2023/2/2
* desc:
*/
public class StrokeManager {
/** Interface to register to be notified of changes in the recognized content. */
public interface ContentChangedListener {
/** This method is called when the recognized content changes. */
void onContentChanged();
}
/** Interface to register to be notified of changes in the status. */
public interface StatusChangedListener {
/** This method is called when the recognized content changes. */
void onStatusChanged();
}
/** Interface to register to be notified of changes in the downloaded model state. */
public interface DownloadedModelsChangedListener {
/** This method is called when the downloaded models changes. */
void onDownloadedModelsChanged(Set<String> downloadedLanguageTags);
}
@VisibleForTesting
static final long CONVERSION_TIMEOUT_MS = 1000;
private static final String TAG = "MLKD.StrokeManager";
// This is a constant that is used as a message identifier to trigger the timeout.
private static final int TIMEOUT_TRIGGER = 1;
// For handling recognition and model downloading.
private RecognitionTask recognitionTask = null;
@VisibleForTesting ModelManager modelManager = new ModelManager();
// Managing the recognition queue.
private final List<RecognitionTask.RecognizedInk> content = new ArrayList<>();
// Managing ink currently drawn.
private Ink.Stroke.Builder strokeBuilder = Ink.Stroke.builder();
private Ink.Builder inkBuilder = Ink.builder();
private boolean stateChangedSinceLastRequest = false;
@Nullable
private ContentChangedListener contentChangedListener = null;
@Nullable private StatusChangedListener statusChangedListener = null;
@Nullable private DownloadedModelsChangedListener downloadedModelsChangedListener = null;
private boolean triggerRecognitionAfterInput = true;
private boolean clearCurrentInkAfterRecognition = true;
private String status = "";
public void setTriggerRecognitionAfterInput(boolean shouldTrigger) {
triggerRecognitionAfterInput = shouldTrigger;
}
public void setClearCurrentInkAfterRecognition(boolean shouldClear) {
clearCurrentInkAfterRecognition = shouldClear;
}
// Handler to handle the UI Timeout.
// This handler is only used to trigger the UI timeout. Each time a UI interaction happens,
// the timer is reset by clearing the queue on this handler and sending a new delayed message (in
// addNewTouchEvent).
private final Handler uiHandler =
new Handler(
msg -> {
if (msg.what == TIMEOUT_TRIGGER) {
Log.i(TAG, "Handling timeout trigger.");
commitResult();
return true;
}
// In the current use this statement is never reached because we only ever send
// TIMEOUT_TRIGGER messages to this handler.
// This line is necessary because otherwise Java's static analysis doesn't allow for
// compiling. Returning false indicates that a message wasn't handled.
return false;
});
private void setStatus(String newStatus) {
status = newStatus;
if (statusChangedListener != null) {
statusChangedListener.onStatusChanged();
}
}
private void commitResult() {
if (recognitionTask.done() && recognitionTask.result() != null) {
content.add(recognitionTask.result());
setStatus("Successful recognition: " + recognitionTask.result().text);
if (clearCurrentInkAfterRecognition) {
resetCurrentInk();
}
if (contentChangedListener != null) {
contentChangedListener.onContentChanged();
}
reset();
}
}
public void reset() {
Log.i(TAG, "reset");
resetCurrentInk();
content.clear();
if (recognitionTask != null && !recognitionTask.done()) {
recognitionTask.cancel();
}
setStatus("");
}
private void resetCurrentInk() {
inkBuilder = Ink.builder();
strokeBuilder = Ink.Stroke.builder();
stateChangedSinceLastRequest = false;
}
public Ink getCurrentInk() {
return inkBuilder.build();
}
/**
* This method is called when a new touch event happens on the drawing client and notifies the
* StrokeManager of new content being added.
*
* <p>This method takes care of triggering the UI timeout and scheduling recognitions on the
* background thread.
*
* @return whether the touch event was handled.
*/
public boolean addNewTouchEvent(MotionEvent event) {
int action = event.getActionMasked();
float x = event.getX();
float y = event.getY();
long t = System.currentTimeMillis();
// A new event happened -> clear all pending timeout messages.
uiHandler.removeMessages(TIMEOUT_TRIGGER);
switch (action) {
case MotionEvent.ACTION_DOWN:
case MotionEvent.ACTION_MOVE:
strokeBuilder.addPoint(Ink.Point.create(x, y, t));
break;
case MotionEvent.ACTION_UP:
strokeBuilder.addPoint(Ink.Point.create(x, y, t));
inkBuilder.addStroke(strokeBuilder.build());
strokeBuilder = Ink.Stroke.builder();
stateChangedSinceLastRequest = true;
recognize();
/* if (triggerRecognitionAfterInput) {
recognize();
}*/
break;
default:
// Indicate touch event wasn't handled.
return false;
}
return true;
}
// Listeners to update the drawing and status.
public void setContentChangedListener(ContentChangedListener contentChangedListener) {
this.contentChangedListener = contentChangedListener;
}
public void setStatusChangedListener(StatusChangedListener statusChangedListener) {
this.statusChangedListener = statusChangedListener;
}
public void setDownloadedModelsChangedListener(
DownloadedModelsChangedListener downloadedModelsChangedListener) {
this.downloadedModelsChangedListener = downloadedModelsChangedListener;
}
public List<RecognitionTask.RecognizedInk> getContent() {
return content;
}
public String getStatus() {
return status;
}
// Model downloading / deleting / setting.
public void setActiveModel(String languageTag) {
setStatus(modelManager.setModel(languageTag));
}
public Task<Void> deleteActiveModel() {
return modelManager
.deleteActiveModel()
.addOnSuccessListener(unused -> refreshDownloadedModelsStatus())
.onSuccessTask(
status -> {
setStatus(status);
return Tasks.forResult(null);
});
}
public Task<Void> download() {
setStatus("Download started.");
return modelManager
.download()
.addOnSuccessListener(unused -> refreshDownloadedModelsStatus())
.onSuccessTask(
status -> {
setStatus(status);
return Tasks.forResult(null);
});
}
// Recognition-related.
public Task<String> recognize() {
if (!stateChangedSinceLastRequest || inkBuilder.isEmpty()) {
setStatus("No recognition, ink unchanged or empty");
return Tasks.forResult(null);
}
if (modelManager.getRecognizer() == null) {
setStatus("Recognizer not set");
return Tasks.forResult(null);
}
return modelManager
.checkIsModelDownloaded()
.onSuccessTask(
result -> {
if (!result) {
setStatus("Model not downloaded yet");
return Tasks.forResult(null);
}
stateChangedSinceLastRequest = false;
recognitionTask =
new RecognitionTask(modelManager.getRecognizer(), inkBuilder.build());
uiHandler.sendMessageDelayed(
uiHandler.obtainMessage(TIMEOUT_TRIGGER), CONVERSION_TIMEOUT_MS);
return recognitionTask.run();
});
}
public void refreshDownloadedModelsStatus() {
modelManager
.getDownloadedModelLanguages()
.addOnSuccessListener(
downloadedLanguageTags -> {
if (downloadedModelsChangedListener != null) {
downloadedModelsChangedListener.onDownloadedModelsChanged(downloadedLanguageTags);
}
});
}
}
3. 创建 ModelManager.java
/**
* author: Kevin-Dev
* date: 2023/2/2
* desc:
*/
public class ModelManager {
private static final String TAG = "MLKD.ModelManager";
private DigitalInkRecognitionModel model;
private DigitalInkRecognizer recognizer;
final RemoteModelManager remoteModelManager = RemoteModelManager.getInstance();
public String setModel(String languageTag) {
// Clear the old model and recognizer.
model = null;
if (recognizer != null) {
recognizer.close();
}
recognizer = null;
// Try to parse the languageTag and get a model from it.
DigitalInkRecognitionModelIdentifier modelIdentifier;
try {
modelIdentifier = DigitalInkRecognitionModelIdentifier.fromLanguageTag(languageTag);
} catch (MlKitException e) {
Log.e(TAG, "Failed to parse language '" + languageTag + "'");
return "";
}
if (modelIdentifier == null) {
return "No model for language: " + languageTag;
}
// Initialize the model and recognizer.
model = DigitalInkRecognitionModel.builder(modelIdentifier).build();
recognizer =
DigitalInkRecognition.getClient(DigitalInkRecognizerOptions.builder(model).build());
Log.i(
TAG,
"Model set for language '"
+ languageTag
+ "' ('"
+ modelIdentifier.getLanguageTag()
+ "').");
return "Model set for language: " + languageTag;
}
public DigitalInkRecognizer getRecognizer() {
return recognizer;
}
public Task<Boolean> checkIsModelDownloaded() {
return remoteModelManager.isModelDownloaded(model);
}
public Task<String> deleteActiveModel() {
if (model == null) {
Log.i(TAG, "Model not set");
return Tasks.forResult("Model not set");
}
return checkIsModelDownloaded()
.onSuccessTask(
result -> {
if (!result) {
return Tasks.forResult("Model not downloaded yet");
}
return remoteModelManager
.deleteDownloadedModel(model)
.onSuccessTask(
aVoid -> {
Log.i(TAG, "Model successfully deleted");
return Tasks.forResult("Model successfully deleted");
});
})
.addOnFailureListener(e -> Log.e(TAG, "Error while model deletion: " + e));
}
public Task<Set<String>> getDownloadedModelLanguages() {
return remoteModelManager
.getDownloadedModels(DigitalInkRecognitionModel.class)
.onSuccessTask(
(remoteModels) -> {
Set<String> result = new HashSet<>();
for (DigitalInkRecognitionModel model : remoteModels) {
result.add(model.getModelIdentifier().getLanguageTag());
}
Log.i(TAG, "Downloaded models for languages:" + result);
return Tasks.forResult(result);
});
}
public Task<String> download() {
if (model == null) {
return Tasks.forResult("Model not selected.");
}
return remoteModelManager
.download(model, new DownloadConditions.Builder().build())
.onSuccessTask(
aVoid -> {
Log.i(TAG, "Model download succeeded.");
return Tasks.forResult("Downloaded model successfully");
})
.addOnFailureListener(e -> Log.e(TAG, "Error while downloading the model: " + e));
}
}
- 创建 RecognitionTask.java
/**
* author: Kevin-Dev
* date: 2023/2/2
* desc:
*/
public class RecognitionTask {
private static final String TAG = "MLKD.RecognitionTask";
private final DigitalInkRecognizer recognizer;
private final Ink ink;
@Nullable
private RecognizedInk currentResult;
private final AtomicBoolean cancelled;
private final AtomicBoolean done;
public RecognitionTask(DigitalInkRecognizer recognizer, Ink ink) {
this.recognizer = recognizer;
this.ink = ink;
this.currentResult = null;
cancelled = new AtomicBoolean(false);
done = new AtomicBoolean(false);
}
public void cancel() {
cancelled.set(true);
}
public boolean done() {
return done.get();
}
@Nullable
public RecognizedInk result() {
return this.currentResult;
}
/** Helper class that stores an ink along with the corresponding recognized text. */
public static class RecognizedInk {
public final Ink ink;
public final String text;
RecognizedInk(Ink ink, String text) {
this.ink = ink;
this.text = text;
}
}
public Task<String> run() {
Log.i(TAG, "RecoTask.run");
return recognizer
.recognize(this.ink)
.onSuccessTask(
result -> {
if (cancelled.get() || result.getCandidates().isEmpty()) {
return Tasks.forResult(null);
}
currentResult = new RecognizedInk(ink, result.getCandidates().get(0).getText());
Log.i(TAG, "result: " + currentResult.text);
done.set(true);
return Tasks.forResult(currentResult.text);
});
}
}
自定义 View
1. DrawingView.java
public class DrawingView extends View implements StrokeManager.ContentChangedListener {
private static final String TAG = "MLKD.DrawingView";
private static final int STROKE_WIDTH_DP = 3;
private static final int MIN_BB_WIDTH = 10;
private static final int MIN_BB_HEIGHT = 10;
private static final int MAX_BB_WIDTH = 256;
private static final int MAX_BB_HEIGHT = 256;
private final Paint recognizedStrokePaint;
private final TextPaint textPaint;
private final Paint currentStrokePaint;
private final Paint canvasPaint;
private final Path currentStroke;
private Canvas drawCanvas;
private Bitmap canvasBitmap;
private StrokeManager strokeManager;
public DrawingView(Context context) {
this(context, null);
}
public DrawingView(Context context, AttributeSet attributeSet) {
super(context, attributeSet);
currentStrokePaint = new Paint();
currentStrokePaint.setColor(Color.BLACK);
currentStrokePaint.setAntiAlias(true);
// Set stroke width based on display density.
currentStrokePaint.setStrokeWidth(
TypedValue.applyDimension(
TypedValue.COMPLEX_UNIT_DIP, STROKE_WIDTH_DP, getResources().getDisplayMetrics()));
currentStrokePaint.setStyle(Paint.Style.STROKE);
currentStrokePaint.setStrokeJoin(Paint.Join.ROUND);
currentStrokePaint.setStrokeCap(Paint.Cap.ROUND);
recognizedStrokePaint = new Paint(currentStrokePaint);
recognizedStrokePaint.setColor(Color.BLACK);
textPaint = new TextPaint();
textPaint.setColor(Color.GREEN);
currentStroke = new Path();
canvasPaint = new Paint(Paint.DITHER_FLAG);
}
private static Rect computeBoundingBox(Ink ink) {
float top = Float.MAX_VALUE;
float left = Float.MAX_VALUE;
float bottom = Float.MIN_VALUE;
float right = Float.MIN_VALUE;
for (Ink.Stroke s : ink.getStrokes()) {
for (Ink.Point p : s.getPoints()) {
top = Math.min(top, p.getY());
left = Math.min(left, p.getX());
bottom = Math.max(bottom, p.getY());
right = Math.max(right, p.getX());
}
}
float centerX = (left + right) / 2;
float centerY = (top + bottom) / 2;
Rect bb = new Rect((int) left, (int) top, (int) right, (int) bottom);
// Enforce a minimum size of the bounding box such that recognitions for small inks are readable
bb.union(
(int) (centerX - MIN_BB_WIDTH / 2),
(int) (centerY - MIN_BB_HEIGHT / 2),
(int) (centerX + MIN_BB_WIDTH / 2),
(int) (centerY + MIN_BB_HEIGHT / 2));
// Enforce a maximum size of the bounding box, to ensure Emoji characters get displayed
// correctly
/*if (bb.width() > MAX_BB_WIDTH) {
bb.set(bb.centerX() - MAX_BB_WIDTH / 2, bb.top, bb.centerX() + MAX_BB_WIDTH / 2, bb.bottom);
}
if (bb.height() > MAX_BB_HEIGHT) {
bb.set(bb.left, bb.centerY() - MAX_BB_HEIGHT / 2, bb.right, bb.centerY() + MAX_BB_HEIGHT / 2);
}*/
return bb;
}
void setStrokeManager(StrokeManager strokeManager) {
this.strokeManager = strokeManager;
}
@Override
protected void onSizeChanged(int width, int height, int oldWidth, int oldHeight) {
Log.i(TAG, "onSizeChanged");
canvasBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
drawCanvas = new Canvas(canvasBitmap);
invalidate();
}
public void redrawContent() {
clear();
Ink currentInk = strokeManager.getCurrentInk();
drawInk(currentInk, currentStrokePaint);
List<RecognitionTask.RecognizedInk> content = strokeManager.getContent();
for (RecognitionTask.RecognizedInk ri : content) {
drawInk(ri.ink, recognizedStrokePaint);
final Rect bb = computeBoundingBox(ri.ink);
drawTextIntoBoundingBox(ri.text, bb, textPaint);
}
invalidate();
}
private void drawTextIntoBoundingBox(String text, Rect bb, TextPaint textPaint) {
final float arbitraryFixedSize = 20.f;
// Set an arbitrary text size to learn how high the text will be.
textPaint.setTextSize(arbitraryFixedSize);
textPaint.setTextScaleX(1.f);
// Now determine the size of the rendered text with these settings.
Rect r = new Rect();
textPaint.getTextBounds(text, 0, text.length(), r);
// Adjust height such that target height is met.
float textSize = arbitraryFixedSize * (float) bb.height() / (float) r.height();
textPaint.setTextSize(textSize);
// Redetermine the size of the rendered text with the new settings.
textPaint.getTextBounds(text, 0, text.length(), r);
// Adjust scaleX to squeeze the text.
textPaint.setTextScaleX((float) bb.width() / (float) r.width());
// And finally draw the text.
drawCanvas.drawText(text, bb.left, bb.bottom, textPaint);
}
private void drawInk(Ink ink, Paint paint) {
for (Ink.Stroke s : ink.getStrokes()) {
drawStroke(s, paint);
}
}
private void drawStroke(Ink.Stroke s, Paint paint) {
Log.i(TAG, "drawstroke");
Path path = null;
for (Ink.Point p : s.getPoints()) {
if (path == null) {
path = new Path();
path.moveTo(p.getX(), p.getY());
} else {
path.lineTo(p.getX(), p.getY());
}
}
drawCanvas.drawPath(path, paint);
}
public void clear() {
currentStroke.reset();
onSizeChanged(
canvasBitmap.getWidth(),
canvasBitmap.getHeight(),
canvasBitmap.getWidth(),
canvasBitmap.getHeight());
}
@Override
protected void onDraw(Canvas canvas) {
canvas.drawBitmap(canvasBitmap, 0, 0, canvasPaint);
canvas.drawPath(currentStroke, currentStrokePaint);
}
@Override
public boolean onTouchEvent(MotionEvent event) {
int action = event.getActionMasked();
float x = event.getX();
float y = event.getY();
switch (action) {
case MotionEvent.ACTION_DOWN:
currentStroke.moveTo(x, y);
break;
case MotionEvent.ACTION_MOVE:
currentStroke.lineTo(x, y);
break;
case MotionEvent.ACTION_UP:
currentStroke.lineTo(x, y);
drawCanvas.drawPath(currentStroke, currentStrokePaint);
currentStroke.reset();
break;
default:
break;
}
strokeManager.addNewTouchEvent(event);
invalidate();
return true;
}
@Override
public void onContentChanged() {
redrawContent();
}
}
2. StatusTextView.java
public class StatusTextView extends TextView implements StrokeManager.StatusChangedListener {
private StrokeManager strokeManager;
public StatusTextView(@NonNull Context context) {
super(context);
}
public StatusTextView(Context context, AttributeSet attributeSet) {
super(context, attributeSet);
}
@Override
public void onStatusChanged() {
this.setText(this.strokeManager.getStatus());
}
void setStrokeManager(StrokeManager strokeManager) {
this.strokeManager = strokeManager;
}
}
使用
1. 布局文件
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:orientation="vertical" android:layout_width="match_parent"
android:layout_height="match_parent">
<com.kjd.gesturedemo.ai.DrawingView
android:id="@+id/drawing_view"
android:layout_width="match_parent"
android:layout_height="200dp"
android:layout_weight="1"
android:background="#80FFFFFF" />
<FrameLayout
android:layout_width="match_parent"
android:layout_height="wrap_content">
<LinearLayout
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical">
<com.kjd.gesturedemo.ai.StatusTextView
android:id="@+id/status_text_view"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text="Status text..."
android:textIsSelectable="true" />
<LinearLayout
android:layout_width="match_parent"
android:layout_height="60dp"
android:orientation="horizontal">
<Button
android:id="@+id/clear_button"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:onClick="clearClick"
android:text="清除" />
</LinearLayout>
</LinearLayout>
</FrameLayout>
</LinearLayout>
2. RecognitionActivity.java
public class RecognitionActivity extends AppCompatActivity implements StrokeManager.DownloadedModelsChangedListener {
private static final String TAG = "MLKDI.Activity";
@VisibleForTesting
final StrokeManager strokeManager = new StrokeManager();
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_recognition);
DrawingView drawingView = findViewById(R.id.drawing_view);
StatusTextView statusTextView = findViewById(R.id.status_text_view);
drawingView.setStrokeManager(strokeManager);
statusTextView.setStrokeManager(strokeManager);
strokeManager.setStatusChangedListener(statusTextView);
strokeManager.setContentChangedListener(drawingView);
strokeManager.setActiveModel("zh-Hani-CN");
strokeManager.setDownloadedModelsChangedListener(this);
strokeManager.setClearCurrentInkAfterRecognition(true);
strokeManager.setTriggerRecognitionAfterInput(false);
strokeManager.download();
strokeManager.recognize();
strokeManager.refreshDownloadedModelsStatus();
strokeManager.reset();
}
public void clearClick(View v) {
strokeManager.reset();
DrawingView drawingView = findViewById(R.id.drawing_view);
drawingView.clear();
}
@Override
public void onDownloadedModelsChanged(Set<String> downloadedLanguageTags) {
}
}