android可见即可说实现方案

• 依赖于科大讯飞的asr识别能力，使用Android无障碍服务获取页面文本作为热词，注册到讯飞api，注册过后语音识别到热词的asr返回,利用WindowManager和无障碍的点击实现可见即可说功能

##  无障碍服务获取需要注册的热词
```
package com..model;import android.accessibilityservice.AccessibilityService;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.ServiceConnection;
import android.graphics.Rect;
import android.os.IBinder;
import android.util.Log;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityNodeInfo;
import androidx.lifecycle.Observer;
import .HotWordsBean;
import .GsonUtil;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;public class MyAccessibilityService extends AccessibilityService implements IAccessibilityHotWord {private String TAG = MyAccessibilityService.class.getSimpleName();private StringBuilder hotWords = new StringBuilder();private VrSpService vrSpeechService;private HotWordsBean hotWordsBean = new HotWordsBean();private AccessibilityNodeInfo rootInActiveWindow;private Set<AccessibilityNodeInfo> accessibilityNodeInfoSet = new HashSet<>();HotWordsBean.UserDataBean userDataBean = new HotWordsBean.UserDataBean();HotWordsBean.UserDataBean.CMD cmd = new HotWordsBean.UserDataBean.CMD();private String hotWordJsonString = "";private Context context;@Overridepublic void onCreate() {super.onCreate();Log.i(TAG, "------------ super.onCreate --------------------: ");bindVrSpeechService();context = this;HotWordReceiver.hotWordLiveData.observeForever( new Observer<String>() {@Overridepublic void onChanged(String hotWord) {Log.d(TAG, "onChanged: ytf  ------------ hotWord change :" + hotWord);if (accessibilityNodeInfoSet.size() > 0) {for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {if (nodeInfo.getText() != null && nodeInfo.getText().toString().equalsIgnoreCase(hotWord)) {Log.d(TAG, "ytf, hotWord Shot:" + hotWord);handlePerformAction(hotWord);}}} else {Log.d(TAG, "ytf hotWord Shot: accessibilityNodeInfoSet size = " + accessibilityNodeInfoSet.size());}}});}@Overridepublic void onAccessibilityEvent(AccessibilityEvent accessibilityEvent) {String packageName = accessibilityEvent.getPackageName() == null ? "" : accessibilityEvent.getPackageName().toString();if (!"com.saicmotor.settings".equals(packageName)) {return;}int eventType = accessibilityEvent.getEventType();
//        Log.d(TAG, "ytf,onAccessibilityEvent [eventType: " + eventType + "], [ packageName: " + packageName + "]");switch (eventType) {
//            case AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED:case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED:case AccessibilityEvent.TYPE_VIEW_CLICKED:accessibilityNodeInfoSet.clear();hotWords.setLength(0);rootInActiveWindow = getRootInActiveWindow();if (null == rootInActiveWindow) {Log.d(TAG, "ytf, onAccessibilityEvent: rootInActiveWindow == null");return;} else {recycle(rootInActiveWindow);}String[] splitHotWords = hotWords.toString().split(",");//vrSpeechService.notifyHotWordLoad(GsonUtil.HOT_WORD_TEST_1);hotWordsBean.setHotWords(splitHotWords);cmd.setActiveStatus("");userDataBean.setCmd(cmd);hotWordsBean.setUserData(userDataBean);try {hotWordJsonString = GsonUtil.getInstance().getGson().toJson(hotWordsBean);if (hotWordJsonString != null) {vrSpeechService.notifyHotWordLoad(hotWordJsonString);Log.d(TAG, "onAccessibilityEvent: ytf, hotWordJsonString = " + hotWordJsonString);}} catch (Exception e) {Log.e(TAG, "onAccessibilityEvent: ytf, e: " + e.toString());}break;default:break;}}private void recycle(AccessibilityNodeInfo info) {if (info.getChildCount() == 0) {if ("android.widget.SeekBar".equals(info.getClassName())) {
//                Class<? extends AccessibilityNodeInfo> aClass = info.getClass();
//                Log.d(TAG, "recycle: ytf aClass = " + aClass.getSimpleName());
//                SeekBar seekBar = (SeekBar) info.getClassName();
//                Log.d(TAG, "recycle: ytf, SeekBar 调节前：" + seekBar.getProgress() + ", getViewIdResourceName:" + info.getViewIdResourceName());
//                Bundle bundle = new Bundle();
//                bundle.putFloat(AccessibilityNodeInfo.ACTION_ARGUMENT_PROGRESS_VALUE, 50.0f);
//                seekBar.performAccessibilityAction(R.id.accessibilityActionSetProgress, bundle);
//                Log.d(TAG, "recycle: ytf, SeekBar 调节后：" + seekBar.getProgress() + ", getViewIdResourceName:" + info.getViewIdResourceName());} else if ("android.widget.ScrollView".contentEquals(info.getClassName())) {
//                Log.d(TAG, "recycle: ytf, android.widget.ScrollView ACTION_SCROLL_FORWARD");
//                info.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD);}
//            Log.i(TAG, "recycle ytf, [ClassName: " + info.getClassName() + "], [Text: " + info.getText() + "], [resId: " + info.getViewIdResourceName() + "]");if (null != info.getText()) {String text = info.getText().toString();hotWords.append(text + ",");accessibilityNodeInfoSet.add(info);}} else {for (int i = 0; i < info.getChildCount(); i++) {if (info.getChild(i) != null) {
//                    Log.d(TAG, "ytf 容器: [" + info.getClassName() + "], [resId:" + info.getViewIdResourceName() + "]");recycle(info.getChild(i));}}}}private void handlePerformAction(String targetHotWord) {Log.d(TAG, "handlePerformAction: ytf, accessibilityNodeInfoSet.size = " + accessibilityNodeInfoSet.size());for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {
//            Log.d(TAG, "ytf handlePerformAction:  nodeInfo.getText().toString() = " + nodeInfo.getText().toString() + ", targetHotWord = " + targetHotWord);if (nodeInfo.getText().toString().equalsIgnoreCase(targetHotWord)) {Log.d(TAG, "ytf, 命中可见即可说 handlePerformAction: " + nodeInfo.getText().toString());forceClick(nodeInfo);
//              nodeInfo.performAction(AccessibilityNodeInfo.ACTION_CLICK);}}}private void forceClick(AccessibilityNodeInfo nodeInfo) {Log.d(TAG, "forceClick: ytf,------------");try {Rect rect = new Rect();nodeInfo.getBoundsInScreen(rect);Log.d(TAG, "ytf, forceClick: " + rect.left + " " + rect.top + " " + rect.right + " " + rect.bottom);int x = (rect.left + rect.right) / 2;int y = (rect.top + rect.bottom) / 2;String cmd = "input tap " + String.valueOf(x) + " " + String.valueOf(y);ProcessBuilder builder = new ProcessBuilder();String[] order = {"input","tap",String.valueOf(x),String.valueOf(y)};try {builder.command(order).start();Log.d(TAG, "ytf, forceClick: [ " + x + ", " + y + "]");} catch (IOException e) {Log.d(TAG, "ytf, forceClick: error: " + e.toString());}} catch (Exception e) {Log.e(TAG, "ytf,error forceClick: " + e.toString());}}@Overridepublic void onInterrupt() {Log.i(TAG, "ytf, onInterrupt");}private void bindVrSpeechService() {Intent intent = new Intent(getApplicationContext(), VrSpeechService.class);boolean result = bindService(intent, serviceConnection, Context.BIND_AUTO_CREATE);Log.d(TAG, "vrSpeechService: ytf bind result:" + result);}private ServiceConnection serviceConnection = new ServiceConnection() {@Overridepublic void onServiceConnected(ComponentName componentName, IBinder iBinder) {try {vrSpeechService = ((VrSpeechService.LocalBinder) iBinder).getService();Log.d(TAG, "onServiceConnected: ytf, vrSpeechService bind ");vrSpeechService.setAccessibilityHotWord((IAccessibilityHotWord) context);} catch (Exception e) {e.printStackTrace();Log.e(TAG, "ytf, onServiceConnected: " + e.toString() );}}@Overridepublic void onServiceDisconnected(ComponentName componentName) {}};@Overridepublic void hotWordShot(String hotWord) {
//        Log.d(TAG, "ytf, hotWordShot: " + hotWord);if (accessibilityNodeInfoSet.size() > 0) {for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {if (nodeInfo.getText().toString().equalsIgnoreCase(hotWord)) {Log.d(TAG, "ytf, hotWordShot:" + hotWord);handlePerformAction(hotWord);}}} else {Log.d(TAG, "ytf hotWordShot: accessibilityNodeInfoSet size = " + accessibilityNodeInfoSet.size());}}
}```

• 清单文件：
  
  
  
  

             <meta-dataandroid:name="android.accessibilityservice"android:resource="@xml/accessibility" /></service><receiver android:name="com.saicmotor.voiceservice.model.HotWordReceiver"android:exported="true"android:enabled="true"><intent-filter android:priority="1000"><action android:name="com.saicmotor.voiceservice.hotword"/></intent-filter></receiver>
  

• @xml/accessibility

<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"android:accessibilityEventTypes="typeAllMask"android:accessibilityFeedbackType="feedbackGeneric"android:canRetrieveWindowContent="true"android:canPerformGestures="true"android:accessibilityFlags="flagReportViewIds"android:notificationTimeout="2000"/>

• 模拟asr热词命中
  /**
  * @Author yangtianfu
  * @Date 2023/9/15 13:05
  * @Describe 监听热词回传执行可见可说
  * adb shell am broadcast -a com.saicmotor.voiceservice.hotword -n com.saicmotor.voiceservice/.model.HotWordReceiver --es hotWord “sound”
  */
  public class HotWordReceiver extends BroadcastReceiver {
  private static final String TAG = “HotWordReceiver”;
  private final String ACTION_HOT_WORD_RECEIVER = “com.saicmotor.voiceservice.hotword”;
  // private IAccessibilityHotWord iAccessibilityHotWord;
  public static MutableLiveData hotWordLiveData = new MutableLiveData<>();

     @Overridepublic void onReceive(Context context, Intent intent) {String action = intent.getAction();Log.d(TAG, "ytf, onReceive: intent,action = " + action);if (ACTION_HOT_WORD_RECEIVER.equals(action)) {String hotWord = intent.getStringExtra("hotWord");Log.d(TAG, "ytf, onReceive: com.saicmotor.voiceservice.hotword :" + hotWord);hotWordLiveData.postValue(hotWord);}}
  

  }

## 科大讯飞注册热词热词格式：public static final String HOT_WORD_TEST_1 = "{\n" +"    \"HotWords\":[\"High\"],\n" +"    \"UserData\":{\n" +"        \"cmd\":{\n" +"            \"activeStatus\":\"bg\",\n" +"            \"data\":{\n" +"\n" +"            },\n" +"            \"sceneStatus\":\"default\"\n" +"        }\n" +"    }\n" +"}";int result = libisssr.uploadData(hotWords, 2);
##  VuiService.java语音Vui服务（带有语音形象的app）

透明activity无法跨应用实现点击穿透效果，会导致可见即可说点击无效果，需要在service中使用WindowManager的addview方法，把语音的app作为view添加到WindowManager中,这样就可以实现语音app全透明状态下识别到asr之后可以利用Android无障碍服务去点击指定位置或者指定控件。public class VoiceVuiService extends Service {private WindowManager.LayoutParams mParams;private WindowManager mWindowManager;private ImageView voiceImage;private TextView textView;@Overridepublic void onCreate() {mParams = new WindowManager.LayoutParams();//设置type.系统提示型窗口，一般都在应用程序窗口之上.mParams.type = WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY;
//        mParams.type = WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;//设置效果为背景透明.
//        mParams.format = PixelFormat.RGBA_8888;mParams.format = PixelFormat.TRANSLUCENT;//设置flags.不可聚焦及不可使用按钮对悬浮窗进行操控.
//        mParams.flags = WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE | WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE |
//                WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL | WindowManager.LayoutParams.FIRST_SYSTEM_WINDOW;mParams.flags = WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE | WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE;mParams.alpha = 0.8f;mWindowManager = (WindowManager) getSystemService(Context.WINDOW_SERVICE);DisplayMetrics dm = new DisplayMetrics();mWindowManager.getDefaultDisplay().getMetrics(dm);mParams.width = 136;mParams.height = 136;
//        mParams.x = 100;
//        mParams.y = 100;voiceImage = new ImageView(this);voiceImage.setBackground(getResources().getDrawable(R.mipmap.assistant100025));textView = new TextView(this);textView.setTextSize(36);textView.setTextColor(Color.RED);textView.setText("语音形象");//        vrView = new HSPortraitVrViewForA11V(getApplicationContext());
//        vrView.startFlipping();Log.d(TAG, "initView: ytf, dm.widthPixels = " + dm.widthPixels + ",mParams.height = " + mParams.height);mParams.gravity = Gravity.TOP;LogUtils.i(TAG, "onCreate.....");super.onCreate();-----------------------@Overridepublic int onStartCommand(Intent intent, int flags, int startId) {LogUtils.i(TAG, "onStartCommand,LteService onStartCommand");// Android 8以上特殊处理setNotificationChannel();
//        mWindowManager.addView(voiceImage, mParams);
//        mWindowManager.addView(textView, mParams);//        if(CommonUtils.isUseHalfServiceMode()){
//            if(offlineAgentService == null){
//                bindHalfEngineService();
//            }
//        }return START_STICKY;}
-------------------------------@Overridepublic void onDestroy() {//        if (voiceImage.getParent() != null) {
//            mWindowManager.removeView(voiceImage);
//        }
//        if (textView.getParent() != null) {
//            mWindowManager.removeView(textView);
//        }super.onDestroy();}