본문 바로가기

드론의 카메라 영상 데이터를 OpenCV에서 활용할 수 있도록 획득하는 예제 코드를 정리해 보았습니다.

아래 경로의 코드를 참고해 주세요.


import android.app.ActionBar;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaFormat;
import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import android.view.TextureView;
import android.view.View;
import android.view.WindowManager;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;

import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;

import dji.common.airlink.PhysicalSource;
import dji.common.camera.SettingsDefinitions;
import dji.common.error.DJIError;
import dji.common.flightcontroller.simulator.SimulatorState;
import dji.common.product.Model;
import dji.common.remotecontroller.GPSData;
import dji.common.remotecontroller.HardwareState;
import dji.common.util.CommonCallbacks;
import dji.sdk.airlink.OcuSyncLink;
import dji.sdk.base.BaseProduct;
import dji.sdk.camera.Camera;
import dji.sdk.camera.VideoFeeder;
import dji.sdk.codec.DJICodecManager;
import dji.sdk.flightcontroller.FlightController;
import dji.sdk.products.Aircraft;
import dji.sdk.remotecontroller.RemoteController;
import dji.sdk.sdkmanager.DJISDKManager;
import io.droneplay.droneplaymission.MApplication;
import io.droneplay.droneplaymission.R;
import io.droneplay.droneplaymission.utils.HelperUtils;
import io.droneplay.droneplaymission.utils.ModuleVerificationUtil;
import io.droneplay.droneplaymission.utils.ToastUtils;
import io.droneplay.droneplaymission.utils.media.DJIVideoStreamDecoder;
import io.droneplay.droneplaymission.utils.media.NativeHelper;


public class OpenCVActivity extends AppCompatActivity implements DJICodecManager.YuvDataCallback {

    private static final String TAG = OpenCVActivity.class.getSimpleName();

    protected VideoFeeder.VideoDataListener mReceivedVideoDataListener = null;
    protected FlightController flightController = null;
    private RemoteController remoteController;
    private Camera mCamera;
    private VideoFeeder.VideoFeed standardVideoFeeder;

    private DJICodecManager mCodecManager;

    private TextureView videostreamPreviewTtView;

    private Context mContext;

    private Mat frameaux = null;
    private Mat framergb = new Mat();

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);

        mContext = this;

        MApplication.getEventBus().register(mContext);

        setContentView(R.layout.activity_catchme);

        if (isM300Product()) {
            OcuSyncLink ocuSyncLink = MApplication.getProductInstance().getAirLink().getOcuSyncLink();
            // If your MutltipleLensCamera is set at right or top, you need to change the PhysicalSource to RIGHT_CAM or TOP_CAM.
            ocuSyncLink.assignSourceToPrimaryChannel(PhysicalSource.LEFT_CAM, PhysicalSource.FPV_CAM, new CommonCallbacks.CompletionCallback() {
                @Override
                public void onResult(DJIError error) {

                }
            });
        }

        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
        initUI(savedInstanceState);
    }

    public static boolean isM300Product() {
        if (DJISDKManager.getInstance().getProduct() == null) {
            return false;
        }
        Model model = DJISDKManager.getInstance().getProduct().getModel();
        return model == Model.MATRICE_300_RTK;
    }

    @Override
    public void onYuvDataReceived(MediaFormat mediaFormat, ByteBuffer byteBuffer, int dataSize, int width, int height) {
        if (dataSize <= 0
                || byteBuffer == null) return;

        final byte[] yuvFrame = new byte[dataSize];
        byteBuffer.get(yuvFrame);

        if (frameaux == null) {
            int currentHeight = height + (height >> 1);
            int currentWidth = width;
            frameaux = new Mat(currentHeight, currentWidth, CvType.CV_8UC1);
        }

        byte[] retFrame;
        if (Build.VERSION.SDK_INT <= 23) {
            retFrame = oldConvertPlanartoYUV(yuvFrame, width, height);
        } else {
            retFrame = newConvertPlanartoYUV(yuvFrame, width, height);
        }
        if (retFrame == null) return;

        frameaux.put(0, 0, retFrame);
        Imgproc.cvtColor(frameaux, framergb, Imgproc.COLOR_YUV2RGB_NV21); // get rgb mat
    }

    private byte[] oldConvertPlanartoYUV(byte[] yuvFrame, int width, int height) {
        if (yuvFrame.length < width * height) {
            return null;
        }

        byte[] y = new byte[width * height];
        byte[] u = new byte[width * height / 4];
        byte[] v = new byte[width * height / 4];
        byte[] nu = new byte[width * height / 4]; //
        byte[] nv = new byte[width * height / 4];

        System.arraycopy(yuvFrame, 0, y, 0, y.length);
        for (int i = 0; i < u.length; i++) {
            v[i] = yuvFrame[y.length + 2 * i];
            u[i] = yuvFrame[y.length + 2 * i + 1];
        }
        int uvWidth = width / 2;
        int uvHeight = height / 2;
        for (int j = 0; j < uvWidth / 2; j++) {
            for (int i = 0; i < uvHeight / 2; i++) {
                byte uSample1 = u[i * uvWidth + j];
                byte uSample2 = u[i * uvWidth + j + uvWidth / 2];
                byte vSample1 = v[(i + uvHeight / 2) * uvWidth + j];
                byte vSample2 = v[(i + uvHeight / 2) * uvWidth + j + uvWidth / 2];
                nu[2 * (i * uvWidth + j)] = uSample1;
                nu[2 * (i * uvWidth + j) + 1] = uSample1;
                nu[2 * (i * uvWidth + j) + uvWidth] = uSample2;
                nu[2 * (i * uvWidth + j) + 1 + uvWidth] = uSample2;
                nv[2 * (i * uvWidth + j)] = vSample1;
                nv[2 * (i * uvWidth + j) + 1] = vSample1;
                nv[2 * (i * uvWidth + j) + uvWidth] = vSample2;
                nv[2 * (i * uvWidth + j) + 1 + uvWidth] = vSample2;
            }
        }
        //nv21test
        byte[] bytes = new byte[yuvFrame.length];
        System.arraycopy(y, 0, bytes, 0, y.length);
        for (int i = 0; i < u.length; i++) {
            bytes[y.length + (i * 2)] = nv[i];
            bytes[y.length + (i * 2) + 1] = nu[i];
        }
        Log.d(TAG,
                "onYuvDataReceived: frame index: "
                        + DJIVideoStreamDecoder.getInstance().frameIndex
                        + ",array length: "
                        + bytes.length);

        return bytes;
    }

    private byte[] newConvertPlanartoYUV(byte[] yuvFrame, int width, int height) {
        if (yuvFrame.length < width * height) {
            //DJILog.d(TAG, "yuvFrame size is too small " + yuvFrame.length);
            return null;
        }
        int length = width * height;

        byte[] u = new byte[width * height / 4];
        byte[] v = new byte[width * height / 4];
        for (int i = 0; i < u.length; i++) {
            v[i] = yuvFrame[length + 2 * i];
            u[i] = yuvFrame[length + 2 * i + 1];
        }
        for (int i = 0; i < u.length; i++) {
            yuvFrame[length + 2 * i] = u[i];
            yuvFrame[length + 2 * i + 1] = v[i];
        }

        return yuvFrame;
    }

    @Override
    protected void onDestroy() {

        if (mCodecManager != null) {
            mCodecManager.cleanSurface();
            mCodecManager.destroyCodec();
        }

        MApplication.getEventBus().unregister(mContext);
        HelperUtils.getInstance().flushMixpanelTracking(mContext);

        super.onDestroy();
    }

    private long lastupdate;

    private void notifyStatusChange() {

        final BaseProduct product = MApplication.getProductInstance();
        // The callback for receiving the raw H264 video data for camera live view
        mReceivedVideoDataListener = new VideoFeeder.VideoDataListener() {
            @Override
            public void onReceive(byte[] videoBuffer, int size) {
                if (System.currentTimeMillis() - lastupdate > 1000) {
                    Log.d(TAG, "camera recv video data size: " + size);
                    lastupdate = System.currentTimeMillis();
                }
                if (mCodecManager != null) {
                    mCodecManager.sendDataToDecoder(videoBuffer, size);
                }

                DJIVideoStreamDecoder.getInstance().parse(videoBuffer, size);
            }
        };

        if (null == product || !product.isConnected()) {
            mCamera = null;
        } else {
            if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)) {
                mCamera = product.getCamera();
                mCamera.setMode(SettingsDefinitions.CameraMode.SHOOT_PHOTO, new CommonCallbacks.CompletionCallback() {
                    @Override
                    public void onResult(DJIError djiError) {
                        if (djiError != null) {
                            //showToast("can't change mode of camera, error:"+djiError.getDescription());
                        }
                    }
                });

                if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)
                        && VideoFeeder.getInstance() != null
                        && VideoFeeder.getInstance().provideTranscodedVideoFeed() != null
                        && isTranscodedVideoFeedNeeded()) {
                    Log.d(TAG, "provideTranscodedVideoFeed  ADDED");
                    standardVideoFeeder = VideoFeeder.getInstance().provideTranscodedVideoFeed();
                    standardVideoFeeder.addVideoDataListener(mReceivedVideoDataListener);
                } else if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)
                        && VideoFeeder.getInstance() != null
                        && VideoFeeder.getInstance().getPrimaryVideoFeed() != null) {
                    Log.d(TAG, "getPrimaryVideoFeed  ADDED");
                    VideoFeeder.getInstance().getPrimaryVideoFeed().addVideoDataListener(mReceivedVideoDataListener);
                } else {
                    if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)
                            && VideoFeeder.getInstance() != null
                            && VideoFeeder.getInstance().getSecondaryVideoFeed() != null) {
                        Log.d(TAG, "getSecondaryVideoFeed  ADDED");
                        VideoFeeder.getInstance().getSecondaryVideoFeed().addVideoDataListener(mReceivedVideoDataListener);
                    }
                }
            }
        }
    }

    @Override
    public void onResume() {
        super.onResume();  // Always call the superclass method first
        initPreviewerTextureView();
        NativeHelper.getInstance().init();
        DJIVideoStreamDecoder.getInstance().init(getApplicationContext(), null);
        DJIVideoStreamDecoder.getInstance().setYuvDataListener(OpenCVActivity.this);
        DJIVideoStreamDecoder.getInstance().resume();

        notifyStatusChange();
    }

    @Override
    public void onPause() {
        super.onPause();  // Always call the superclass method first

        if (mCamera != null) {
            if (VideoFeeder.getInstance().getPrimaryVideoFeed() != null) {
                VideoFeeder.getInstance().getPrimaryVideoFeed().removeVideoDataListener(mReceivedVideoDataListener);
            }
            if (standardVideoFeeder != null) {
                standardVideoFeeder.removeVideoDataListener(mReceivedVideoDataListener);
            }
        }

        DJIVideoStreamDecoder.getInstance().stop();
        NativeHelper.getInstance().release();
    }


    @Override
    public void onStart() {
        super.onStart();

        setUpListener();
    }

    @Override
    public void onStop() {
        super.onStop();

        if (flightController != null) {
            flightController.getSimulator().stop(null);
            flightController.setStateCallback(null);
        }
    }

    private void initUI(Bundle savedInstanceState) {
        View decorView = getWindow().getDecorView();
        int uiOptions = View.SYSTEM_UI_FLAG_FULLSCREEN;
        decorView.setSystemUiVisibility(uiOptions);
        ActionBar actionBar = getActionBar();
        if (actionBar != null)
            actionBar.hide();

        HelperUtils.getInstance().hideSystemUI(OpenCVActivity.this);

        videostreamPreviewTtView = findViewById(R.id.livestream_preview_ttv);
    }

    private void initPreviewerTextureView() {
        videostreamPreviewTtView.setSurfaceTextureListener(new TextureView.SurfaceTextureListener() {
            @Override
            public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) {
                if (mCodecManager == null) {
                    mCodecManager = new DJICodecManager(getApplicationContext(), surface, width, height);
                    //For M300RTK, you need to actively request an I frame.
                    mCodecManager.resetKeyFrame();
                }
            }

            @Override
            public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) {

            }

            @Override
            public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
                if (mCodecManager != null) {
                    mCodecManager.cleanSurface();
                }
                return false;
            }

            @Override
            public void onSurfaceTextureUpdated(SurfaceTexture surface) {

            }
        });
    }

    private boolean isTranscodedVideoFeedNeeded() {
        if (VideoFeeder.getInstance() == null) {
            return false;
        }

        return VideoFeeder.getInstance().isFetchKeyFrameNeeded() || VideoFeeder.getInstance()
                .isLensDistortionCalibrationNeeded();
    }

    private void setUpListener() {
        BaseProduct product = MApplication.getProductInstance();

        if (product == null || !product.isConnected()) {
            ToastUtils.setResultToToast("Disconnected");
            return;
        } else {
            if (ModuleVerificationUtil.isRemoteControllerAvailable()) {
                remoteController = ((Aircraft) product).getRemoteController();
                if (remoteController == null) {
                    ToastUtils.setResultToToast("Remotecontroller is not connected");
                    return;
                }

                remoteController.setGPSDataCallback(new GPSData.Callback() {
                    @Override
                    public void onUpdate(@NonNull GPSData gpsData) {

                    }
                });

                remoteController.setHardwareStateCallback(new HardwareState.HardwareStateCallback() {
                    @Override
                    public void onUpdate(@NonNull HardwareState rcHardwareState) {

                    }
                });
            }

            if (flightController != null) return;

            if (product instanceof Aircraft) {
                flightController = ((Aircraft) product).getFlightController();
                flightController.getSimulator().setStateCallback(new SimulatorState.Callback() {
                    @Override
                    public void onUpdate(final SimulatorState stateData) {
                        new Handler(Looper.getMainLooper()).post(new Runnable() {
                            @Override
                            public void run() {

                                String yaw = String.format("%.2f", stateData.getYaw());
                                String pitch = String.format("%.2f", stateData.getPitch());
                                String roll = String.format("%.2f", stateData.getRoll());
                                String positionX = String.format("%.2f", stateData.getPositionX());
                                String positionY = String.format("%.2f", stateData.getPositionY());
                                String positionZ = String.format("%.2f", stateData.getPositionZ());

                                Log.d(TAG, "Yaw : " + yaw + ", Pitch : " + pitch + ", Roll : " + roll + "\n" + ", PosX : " + positionX +
                                        ", PosY : " + positionY +
                                        ", PosZ : " + positionZ);
                            }
                        });
                    }
                });

                flightController.setVirtualStickModeEnabled(true, new CommonCallbacks.CompletionCallback() {
                    @Override
                    public void onResult(DJIError djiError) {
                        if (djiError != null) {
                            Log.d(TAG, djiError.getDescription());
                        } else {
                            Log.d(TAG, "Enable Virtual Stick Success");
                        }
                    }
                });
            }
        }
    }
}








번호 제목 날짜 조회 수
공지 [TIP] 안드로이드 앱 빌드시 "Error:Execution failed for task ':app:compileDebugJavaWithJavac'" 오류가 발생할 경우 2017.09.13 26331
공지 [TIP] 안드로이드 - 설치된 앱 목록 얻기 2016.03.22 21953
공지 [TIP] C# Windows Mobile 용 HttpUtility 코드 2010.02.23 18377
» [TIP] DJI Mobile SDK + OpenCV 적용 방법 2021.10.30 1467
46 [TIP] React-native 빌드 오류 대응 방법 2021.09.10 2592
45 [TIP] 안드로이드 프로젝트 열 때 Failed to apply plugin [id 'com.android.application'] 오류 2021.07.03 785
44 [TIP] 안드로이드 앱이 처음 설치될때 Referrer 정보 받아 오기 (2020.3.1 이후) [2] 2020.10.15 1960
43 tizen .net wearable widget 질문해도 될까요? [1] 2019.08.19 690
42 [TIP] iOS - UIWebView에 로컬 html 파일 로드하기 (Swift 4) 2019.06.05 3048
41 [TIP] Tizen Push가 갑자기 내려오지 않을때 2019.05.20 984
40 [TIP] 두근두근앱이 라즈베리파이를 두근거리도록 개발한 기록 #3 2019.05.15 3687
39 [TIP] 두근두근앱이 라즈베리파이를 두근거리도록 개발한 기록 #2 [4] 2019.05.11 2962
38 [TIP] Android 코드에서 블루투스 연결이 잘 안될때 [7] 2019.05.11 8796
37 [TIP] 두근두근앱이 라즈베리파이를 두근거리도록 개발한 기록 #1 [2] 2019.05.07 6856
36 [TIP] 타이젠 스튜디오에서 웨어러블 디바이스로 디버깅을 위한 바이너리 전송이 안 될때 2019.03.26 1836
35 [TIP] http://tizen.org/system/tizenid 으로 타이젠 고유 id를 확보할 때 유의할 점 2019.03.14 1006
34 [TIP] "cordova run android" 명령을 실행했는데 "A problem occurred evaluating project ':CordovaLib'"오류가 뜰때 2019.01.22 2426
33 [TIP] Mac에서 Cordova run android 를 실행했는데 "Command failed with exit code EACCES" 오류가 뜰때 2019.01.22 1584
32 [TIP] 안드로이드에서 심박수 측정하는 코드 [28] 2019.01.02 7500
31 [TIP] Google Cloud API 사용시 안드로이드의 Assets 폴더에 있는 Crendential 파일 사용하기 2018.05.29 1175
30 [TIP] 안드로이드 스튜디오에서 Error:android-apt plugin is incompatible with the Android Gradle plugin. Please use 'annotationProcessor' configuration instead 오류날때 2018.02.02 4494
29 [TIP] 안드로이드 앱 빌드시 "Error:Execution failed for task ':app:compileDebugJavaWithJavac'" 오류가 발생할 경우 2017.09.13 26331
28 [TIP] Error:java.lang.OutOfMemoryError: GC overhead limit exceeded 2016.11.03 1715