드론의 카메라 영상 데이터를 OpenCV에서 활용할 수 있도록 획득하는 예제 코드를 정리해 보았습니다.
아래 경로의 코드를 참고해 주세요.
import android.app.ActionBar;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaFormat;
import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import android.view.TextureView;
import android.view.View;
import android.view.WindowManager;
import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import dji.common.airlink.PhysicalSource;
import dji.common.camera.SettingsDefinitions;
import dji.common.error.DJIError;
import dji.common.flightcontroller.simulator.SimulatorState;
import dji.common.product.Model;
import dji.common.remotecontroller.GPSData;
import dji.common.remotecontroller.HardwareState;
import dji.common.util.CommonCallbacks;
import dji.sdk.airlink.OcuSyncLink;
import dji.sdk.base.BaseProduct;
import dji.sdk.camera.Camera;
import dji.sdk.camera.VideoFeeder;
import dji.sdk.codec.DJICodecManager;
import dji.sdk.flightcontroller.FlightController;
import dji.sdk.products.Aircraft;
import dji.sdk.remotecontroller.RemoteController;
import dji.sdk.sdkmanager.DJISDKManager;
import io.droneplay.droneplaymission.MApplication;
import io.droneplay.droneplaymission.R;
import io.droneplay.droneplaymission.utils.HelperUtils;
import io.droneplay.droneplaymission.utils.ModuleVerificationUtil;
import io.droneplay.droneplaymission.utils.ToastUtils;
import io.droneplay.droneplaymission.utils.media.DJIVideoStreamDecoder;
import io.droneplay.droneplaymission.utils.media.NativeHelper;
public class OpenCVActivity extends AppCompatActivity implements DJICodecManager.YuvDataCallback {
private static final String TAG = OpenCVActivity.class.getSimpleName();
protected VideoFeeder.VideoDataListener mReceivedVideoDataListener = null;
protected FlightController flightController = null;
private RemoteController remoteController;
private Camera mCamera;
private VideoFeeder.VideoFeed standardVideoFeeder;
private DJICodecManager mCodecManager;
private TextureView videostreamPreviewTtView;
private Context mContext;
private Mat frameaux = null;
private Mat framergb = new Mat();
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
mContext = this;
MApplication.getEventBus().register(mContext);
setContentView(R.layout.activity_catchme);
if (isM300Product()) {
OcuSyncLink ocuSyncLink = MApplication.getProductInstance().getAirLink().getOcuSyncLink();
// If your MutltipleLensCamera is set at right or top, you need to change the PhysicalSource to RIGHT_CAM or TOP_CAM.
ocuSyncLink.assignSourceToPrimaryChannel(PhysicalSource.LEFT_CAM, PhysicalSource.FPV_CAM, new CommonCallbacks.CompletionCallback() {
@Override
public void onResult(DJIError error) {
}
});
}
getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
initUI(savedInstanceState);
}
public static boolean isM300Product() {
if (DJISDKManager.getInstance().getProduct() == null) {
return false;
}
Model model = DJISDKManager.getInstance().getProduct().getModel();
return model == Model.MATRICE_300_RTK;
}
@Override
public void onYuvDataReceived(MediaFormat mediaFormat, ByteBuffer byteBuffer, int dataSize, int width, int height) {
if (dataSize <= 0
|| byteBuffer == null) return;
final byte[] yuvFrame = new byte[dataSize];
byteBuffer.get(yuvFrame);
if (frameaux == null) {
int currentHeight = height + (height >> 1);
int currentWidth = width;
frameaux = new Mat(currentHeight, currentWidth, CvType.CV_8UC1);
}
byte[] retFrame;
if (Build.VERSION.SDK_INT <= 23) {
retFrame = oldConvertPlanartoYUV(yuvFrame, width, height);
} else {
retFrame = newConvertPlanartoYUV(yuvFrame, width, height);
}
if (retFrame == null) return;
frameaux.put(0, 0, retFrame);
Imgproc.cvtColor(frameaux, framergb, Imgproc.COLOR_YUV2RGB_NV21); // get rgb mat
}
private byte[] oldConvertPlanartoYUV(byte[] yuvFrame, int width, int height) {
if (yuvFrame.length < width * height) {
return null;
}
byte[] y = new byte[width * height];
byte[] u = new byte[width * height / 4];
byte[] v = new byte[width * height / 4];
byte[] nu = new byte[width * height / 4]; //
byte[] nv = new byte[width * height / 4];
System.arraycopy(yuvFrame, 0, y, 0, y.length);
for (int i = 0; i < u.length; i++) {
v[i] = yuvFrame[y.length + 2 * i];
u[i] = yuvFrame[y.length + 2 * i + 1];
}
int uvWidth = width / 2;
int uvHeight = height / 2;
for (int j = 0; j < uvWidth / 2; j++) {
for (int i = 0; i < uvHeight / 2; i++) {
byte uSample1 = u[i * uvWidth + j];
byte uSample2 = u[i * uvWidth + j + uvWidth / 2];
byte vSample1 = v[(i + uvHeight / 2) * uvWidth + j];
byte vSample2 = v[(i + uvHeight / 2) * uvWidth + j + uvWidth / 2];
nu[2 * (i * uvWidth + j)] = uSample1;
nu[2 * (i * uvWidth + j) + 1] = uSample1;
nu[2 * (i * uvWidth + j) + uvWidth] = uSample2;
nu[2 * (i * uvWidth + j) + 1 + uvWidth] = uSample2;
nv[2 * (i * uvWidth + j)] = vSample1;
nv[2 * (i * uvWidth + j) + 1] = vSample1;
nv[2 * (i * uvWidth + j) + uvWidth] = vSample2;
nv[2 * (i * uvWidth + j) + 1 + uvWidth] = vSample2;
}
}
//nv21test
byte[] bytes = new byte[yuvFrame.length];
System.arraycopy(y, 0, bytes, 0, y.length);
for (int i = 0; i < u.length; i++) {
bytes[y.length + (i * 2)] = nv[i];
bytes[y.length + (i * 2) + 1] = nu[i];
}
Log.d(TAG,
"onYuvDataReceived: frame index: "
+ DJIVideoStreamDecoder.getInstance().frameIndex
+ ",array length: "
+ bytes.length);
return bytes;
}
private byte[] newConvertPlanartoYUV(byte[] yuvFrame, int width, int height) {
if (yuvFrame.length < width * height) {
//DJILog.d(TAG, "yuvFrame size is too small " + yuvFrame.length);
return null;
}
int length = width * height;
byte[] u = new byte[width * height / 4];
byte[] v = new byte[width * height / 4];
for (int i = 0; i < u.length; i++) {
v[i] = yuvFrame[length + 2 * i];
u[i] = yuvFrame[length + 2 * i + 1];
}
for (int i = 0; i < u.length; i++) {
yuvFrame[length + 2 * i] = u[i];
yuvFrame[length + 2 * i + 1] = v[i];
}
return yuvFrame;
}
@Override
protected void onDestroy() {
if (mCodecManager != null) {
mCodecManager.cleanSurface();
mCodecManager.destroyCodec();
}
MApplication.getEventBus().unregister(mContext);
HelperUtils.getInstance().flushMixpanelTracking(mContext);
super.onDestroy();
}
private long lastupdate;
private void notifyStatusChange() {
final BaseProduct product = MApplication.getProductInstance();
// The callback for receiving the raw H264 video data for camera live view
mReceivedVideoDataListener = new VideoFeeder.VideoDataListener() {
@Override
public void onReceive(byte[] videoBuffer, int size) {
if (System.currentTimeMillis() - lastupdate > 1000) {
Log.d(TAG, "camera recv video data size: " + size);
lastupdate = System.currentTimeMillis();
}
if (mCodecManager != null) {
mCodecManager.sendDataToDecoder(videoBuffer, size);
}
DJIVideoStreamDecoder.getInstance().parse(videoBuffer, size);
}
};
if (null == product || !product.isConnected()) {
mCamera = null;
} else {
if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)) {
mCamera = product.getCamera();
mCamera.setMode(SettingsDefinitions.CameraMode.SHOOT_PHOTO, new CommonCallbacks.CompletionCallback() {
@Override
public void onResult(DJIError djiError) {
if (djiError != null) {
//showToast("can't change mode of camera, error:"+djiError.getDescription());
}
}
});
if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)
&& VideoFeeder.getInstance() != null
&& VideoFeeder.getInstance().provideTranscodedVideoFeed() != null
&& isTranscodedVideoFeedNeeded()) {
Log.d(TAG, "provideTranscodedVideoFeed ADDED");
standardVideoFeeder = VideoFeeder.getInstance().provideTranscodedVideoFeed();
standardVideoFeeder.addVideoDataListener(mReceivedVideoDataListener);
} else if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)
&& VideoFeeder.getInstance() != null
&& VideoFeeder.getInstance().getPrimaryVideoFeed() != null) {
Log.d(TAG, "getPrimaryVideoFeed ADDED");
VideoFeeder.getInstance().getPrimaryVideoFeed().addVideoDataListener(mReceivedVideoDataListener);
} else {
if (!product.getModel().equals(Model.UNKNOWN_AIRCRAFT)
&& VideoFeeder.getInstance() != null
&& VideoFeeder.getInstance().getSecondaryVideoFeed() != null) {
Log.d(TAG, "getSecondaryVideoFeed ADDED");
VideoFeeder.getInstance().getSecondaryVideoFeed().addVideoDataListener(mReceivedVideoDataListener);
}
}
}
}
}
@Override
public void onResume() {
super.onResume(); // Always call the superclass method first
initPreviewerTextureView();
NativeHelper.getInstance().init();
DJIVideoStreamDecoder.getInstance().init(getApplicationContext(), null);
DJIVideoStreamDecoder.getInstance().setYuvDataListener(OpenCVActivity.this);
DJIVideoStreamDecoder.getInstance().resume();
notifyStatusChange();
}
@Override
public void onPause() {
super.onPause(); // Always call the superclass method first
if (mCamera != null) {
if (VideoFeeder.getInstance().getPrimaryVideoFeed() != null) {
VideoFeeder.getInstance().getPrimaryVideoFeed().removeVideoDataListener(mReceivedVideoDataListener);
}
if (standardVideoFeeder != null) {
standardVideoFeeder.removeVideoDataListener(mReceivedVideoDataListener);
}
}
DJIVideoStreamDecoder.getInstance().stop();
NativeHelper.getInstance().release();
}
@Override
public void onStart() {
super.onStart();
setUpListener();
}
@Override
public void onStop() {
super.onStop();
if (flightController != null) {
flightController.getSimulator().stop(null);
flightController.setStateCallback(null);
}
}
private void initUI(Bundle savedInstanceState) {
View decorView = getWindow().getDecorView();
int uiOptions = View.SYSTEM_UI_FLAG_FULLSCREEN;
decorView.setSystemUiVisibility(uiOptions);
ActionBar actionBar = getActionBar();
if (actionBar != null)
actionBar.hide();
HelperUtils.getInstance().hideSystemUI(OpenCVActivity.this);
videostreamPreviewTtView = findViewById(R.id.livestream_preview_ttv);
}
private void initPreviewerTextureView() {
videostreamPreviewTtView.setSurfaceTextureListener(new TextureView.SurfaceTextureListener() {
@Override
public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) {
if (mCodecManager == null) {
mCodecManager = new DJICodecManager(getApplicationContext(), surface, width, height);
//For M300RTK, you need to actively request an I frame.
mCodecManager.resetKeyFrame();
}
}
@Override
public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) {
}
@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
if (mCodecManager != null) {
mCodecManager.cleanSurface();
}
return false;
}
@Override
public void onSurfaceTextureUpdated(SurfaceTexture surface) {
}
});
}
private boolean isTranscodedVideoFeedNeeded() {
if (VideoFeeder.getInstance() == null) {
return false;
}
return VideoFeeder.getInstance().isFetchKeyFrameNeeded() || VideoFeeder.getInstance()
.isLensDistortionCalibrationNeeded();
}
private void setUpListener() {
BaseProduct product = MApplication.getProductInstance();
if (product == null || !product.isConnected()) {
ToastUtils.setResultToToast("Disconnected");
return;
} else {
if (ModuleVerificationUtil.isRemoteControllerAvailable()) {
remoteController = ((Aircraft) product).getRemoteController();
if (remoteController == null) {
ToastUtils.setResultToToast("Remotecontroller is not connected");
return;
}
remoteController.setGPSDataCallback(new GPSData.Callback() {
@Override
public void onUpdate(@NonNull GPSData gpsData) {
}
});
remoteController.setHardwareStateCallback(new HardwareState.HardwareStateCallback() {
@Override
public void onUpdate(@NonNull HardwareState rcHardwareState) {
}
});
}
if (flightController != null) return;
if (product instanceof Aircraft) {
flightController = ((Aircraft) product).getFlightController();
flightController.getSimulator().setStateCallback(new SimulatorState.Callback() {
@Override
public void onUpdate(final SimulatorState stateData) {
new Handler(Looper.getMainLooper()).post(new Runnable() {
@Override
public void run() {
String yaw = String.format("%.2f", stateData.getYaw());
String pitch = String.format("%.2f", stateData.getPitch());
String roll = String.format("%.2f", stateData.getRoll());
String positionX = String.format("%.2f", stateData.getPositionX());
String positionY = String.format("%.2f", stateData.getPositionY());
String positionZ = String.format("%.2f", stateData.getPositionZ());
Log.d(TAG, "Yaw : " + yaw + ", Pitch : " + pitch + ", Roll : " + roll + "\n" + ", PosX : " + positionX +
", PosY : " + positionY +
", PosZ : " + positionZ);
}
});
}
});
flightController.setVirtualStickModeEnabled(true, new CommonCallbacks.CompletionCallback() {
@Override
public void onResult(DJIError djiError) {
if (djiError != null) {
Log.d(TAG, djiError.getDescription());
} else {
Log.d(TAG, "Enable Virtual Stick Success");
}
}
});
}
}
}
}