diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index cacfc1ba..7261f9ac 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -165,6 +165,11 @@ public static int getChannel(Context context) { Context.MODE_PRIVATE).getInt("wifi-channel", 161); } + public static boolean getLowLatencySetting(Context context) { + return context.getSharedPreferences("general", + Context.MODE_PRIVATE).getBoolean("low_latency_decoder", true); + } + public static int getBandwidth(Context context) { return context.getSharedPreferences("general", Context.MODE_PRIVATE).getInt("bandwidth", 20); @@ -353,6 +358,7 @@ private void initializeWfbNg() { private void initializeVideoPlayers() { videoPlayer = new VideoPlayer(this); videoPlayer.setIVideoParamsChanged(this); + videoPlayer.setLowLatency(getLowLatencySetting(this)); isVRMode = getVRSetting(); @@ -587,6 +593,9 @@ private void showSettingsMenu(View anchor) { // Bandwidth submenu setupBandwidthSubMenu(popup); + // Video submenu + setupVideoSubMenu(popup); + // OSD submenu setupOSDSubMenu(popup); @@ -671,6 +680,32 @@ private void setupBandwidthSubMenu(PopupMenu popup) { } } + /** + * Submenu for video decoder options. + * "Low latency" sets the MediaCodec low-latency and realtime-priority keys. It is on + * by default; decoders that misbehave with those keys can be put back on the stock + * pipeline here. + */ + private void setupVideoSubMenu(PopupMenu popup) { + SubMenu videoMenu = popup.getMenu().addSubMenu("Video"); + + MenuItem lowLatencyItem = videoMenu.add("Low latency"); + lowLatencyItem.setCheckable(true); + lowLatencyItem.setChecked(getLowLatencySetting(this)); + lowLatencyItem.setOnMenuItemClickListener(item -> { + boolean enabled = !item.isChecked(); + item.setChecked(enabled); + getSharedPreferences("general", MODE_PRIVATE).edit() + .putBoolean("low_latency_decoder", enabled).apply(); + videoPlayer.setLowLatency(enabled); + Toast.makeText(this, "Low latency " + (enabled ? "enabled" : "disabled") + + ", applies on next video start.", Toast.LENGTH_SHORT).show(); + item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW); + item.setActionView(new View(this)); + return false; + }); + } + /** * Submenu handling OSD toggles and locks. */ diff --git a/app/videonative/src/main/cpp/VideoDecoder.cpp b/app/videonative/src/main/cpp/VideoDecoder.cpp index 47d1079a..0ae7f05d 100644 --- a/app/videonative/src/main/cpp/VideoDecoder.cpp +++ b/app/videonative/src/main/cpp/VideoDecoder.cpp @@ -135,15 +135,6 @@ void VideoDecoder::configureStartDecoder(int idx) AMediaFormat* format = AMediaFormat_new(); AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, MIME.c_str()); - // AMediaFormat_setInt32(format, "low-latency", 1); - // AMediaFormat_setInt32(format, "vendor.low-latency.enable", 1); - // AMediaFormat_setInt32(format, "vendor.qti-ext-dec-low-latency.enable", 1); - // AMediaFormat_setInt32(format, "vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", 1); - // AMediaFormat_setInt32(format, "vendor.rtc-ext-dec-low-latency.enable", 1); - - // MediaCodec supports two priorities: 0 - realtime, 1 - best effort - // AMediaFormat_setInt32(format, "priority", 0); - if (IS_H265) { h265_configureAMediaFormat(mKeyFrameFinder, format); @@ -153,6 +144,11 @@ void VideoDecoder::configureStartDecoder(int idx) h264_configureAMediaFormat(mKeyFrameFinder, format); } + if (mLowLatency) + { + writeAndroidPerformanceParams(format); + } + MLOGD << "Configuring decoder:" << AMediaFormat_toString(format); auto status = AMediaCodec_configure(decoder.codec[idx], format, decoder.window[idx], nullptr, 0); diff --git a/app/videonative/src/main/cpp/VideoDecoder.h b/app/videonative/src/main/cpp/VideoDecoder.h index bf815a2e..a20b296a 100644 --- a/app/videonative/src/main/cpp/VideoDecoder.h +++ b/app/videonative/src/main/cpp/VideoDecoder.h @@ -86,6 +86,10 @@ class VideoDecoder void registerOnDecodingInfoChangedCallback(DECODING_INFO_CHANGED_CALLBACK decodingInfoChangedCallback); + // Enable / disable the low-latency and realtime-priority AMediaFormat keys. + // Applied the next time the decoder is configured, not to a running decoder. + void setLowLatency(bool enabled) { mLowLatency = enabled; } + // If the decoder has been configured, feed NALU. Else search for configuration data and // configure as soon as possible // If the input pipe was closed (surface has been removed or is not set yet), only buffer key frames @@ -109,6 +113,8 @@ class VideoDecoder std::unique_ptr mCheckOutputThread[2] = {nullptr, nullptr}; bool USE_SW_DECODER_INSTEAD = false; + // Some decoders misbehave with the low-latency keys, so it stays user switchable. + std::atomic mLowLatency = true; // Holds the AMediaCodec instance, as well as the state (configured or not configured) Decoder decoder{}; DecodingInfo decodingInfo; diff --git a/app/videonative/src/main/cpp/VideoPlayer.cpp b/app/videonative/src/main/cpp/VideoPlayer.cpp index 6175d585..5a106a3c 100644 --- a/app/videonative/src/main/cpp/VideoPlayer.cpp +++ b/app/videonative/src/main/cpp/VideoPlayer.cpp @@ -318,6 +318,16 @@ extern "C" } } + JNI_METHOD(void, nativeSetLowLatency) + (JNIEnv* env, jclass jclass1, jlong nativeInstance, jboolean enabled) + { + VideoPlayer* p = native(nativeInstance); + if (p) + { + p->setLowLatency(enabled); + } + } + JNI_METHOD(void, nativeSetVideoSurface) (JNIEnv* env, jclass jclass1, jlong videoPlayerN, jobject surface, jint index) { diff --git a/app/videonative/src/main/cpp/VideoPlayer.h b/app/videonative/src/main/cpp/VideoPlayer.h index 830d423e..f339b1fc 100644 --- a/app/videonative/src/main/cpp/VideoPlayer.h +++ b/app/videonative/src/main/cpp/VideoPlayer.h @@ -54,6 +54,8 @@ class VideoPlayer void setForwarding(const std::string& ip, int port, bool enabled); + void setLowLatency(bool enabled) { videoDecoder.setLowLatency(enabled); } + private: void onNewNALU(const NALU& nalu); diff --git a/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h b/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h index 28a05897..bc3ffa9d 100644 --- a/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h +++ b/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h @@ -5,26 +5,22 @@ #include #include "../NALU/KeyFrameFinder.hpp" -// Some of these params are only supported on the latest Android versions -// However,writing them has no negative affect on devices with older Android versions -// Note that for example the low-latency key cannot fix any issues like the 'VUI' issue -void writeAndroidPerformanceParams(AMediaFormat* format) +// Decoder tuning that trades pipeline depth for latency. Unknown keys are ignored by +// MediaCodec, so writing all of them is safe on every device / Android version. +static void writeAndroidPerformanceParams(AMediaFormat* format) { - // I think: KEY_LOW_LATENCY is for decoder. But it doesn't really make a difference anyways - static const auto PARAMETER_KEY_LOW_LATENCY = "low-latency"; - AMediaFormat_setInt32(format, PARAMETER_KEY_LOW_LATENCY, 1); - // Lower values mean higher priority - // Works on pixel 3 (look at output format description) - static const auto AMEDIAFORMAT_KEY_PRIORITY = "priority"; - AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_PRIORITY, 0); - // set operating rate ? - doesn't make a difference - // static const auto AMEDIAFORMAT_KEY_OPERATING_RATE="operating-rate"; - // AMediaFormat_setInt32(format,AMEDIAFORMAT_KEY_OPERATING_RATE,60); - // - // AMEDIAFORMAT_KEY_LOW_LATENCY; - // AMEDIAFORMAT_KEY_LATENCY; - // AMediaFormat_setInt32(format,AMEDIAFORMAT_KEY_LATENCY,0); - // AMediaFormat_setInt32(format,AMEDIAFORMAT_KEY_OPERATING_RATE,0); + // AMEDIAFORMAT_KEY_LOW_LATENCY (API 30+). Tells the decoder to output a frame as soon + // as it is decoded instead of keeping a reorder/output queue. For a live stream that + // never uses B-frames the queue only adds latency. + AMediaFormat_setInt32(format, "low-latency", 1); + // Vendor equivalents for SoCs whose codec does not pick up the AOSP key. Qualcomm is + // the relevant one for most phones and for the Snapdragon XR2 headsets. + AMediaFormat_setInt32(format, "vendor.low-latency.enable", 1); + AMediaFormat_setInt32(format, "vendor.qti-ext-dec-low-latency.enable", 1); + AMediaFormat_setInt32(format, "vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", 1); + AMediaFormat_setInt32(format, "vendor.rtc-ext-dec-low-latency.enable", 1); + // MediaCodec knows two priorities: 0 - realtime, 1 - best effort. Lower is higher. + AMediaFormat_setInt32(format, "priority", 0); } static void h264_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format) @@ -42,7 +38,6 @@ static void h264_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format // AVCProfileBaseline==1 // AMediaFormat_setInt32(decoder.format,AMEDIAFORMAT_KEY_PROFILE,1); // AMediaFormat_setInt32(decoder.format,AMEDIAFORMAT_KEY_PRIORITY,0); - // writeAndroidPerformanceParams(format); } static void h265_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format) @@ -60,7 +55,6 @@ static void h265_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, videoWH[1]); AMediaFormat_setBuffer(format, "csd-0", buff.data(), buff.size()); MLOGD << "Video WH:" << videoWH[0] << " H:" << videoWH[1]; - // writeAndroidPerformanceParams(format); } #endif // FPVUE_ANDROIDMEDIAFORMATHELPER_H diff --git a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java index 7330eb8b..e1c49d0f 100644 --- a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java +++ b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java @@ -52,6 +52,8 @@ public VideoPlayer(final AppCompatActivity parent) { public static native void nativeSetUdpForwarding(long nativeInstance, String ip, int port, boolean enabled); + public static native void nativeSetLowLatency(long nativeInstance, boolean enabled); + public static native void nativeStartDvr(long nativeInstance, int fd, int fmp4_enabled); public static native void nativeStopDvr(long nativeInstance); @@ -126,6 +128,14 @@ public boolean isRunning() { return timer != null; } + /** + * Enable/disable the low latency + realtime priority MediaCodec keys. + * Takes effect the next time the decoder is configured. + */ + public void setLowLatency(boolean enabled) { + nativeSetLowLatency(nativeVideoPlayer, enabled); + } + public void setUdpForwarding(String ip, int port, boolean enabled) { verifyApplicationThread(); nativeSetUdpForwarding(nativeVideoPlayer, ip, port, enabled);