Fix: Generate thumbnail by considering the rotation

mediainfo/src/main/cpp/frame_extractor.cpp

@@ -3,13 +3,15 @@ extern "C" {
 #include <libavcodec/avcodec.h>
 #include <libswscale/swscale.h>
 #include <libavutil/imgutils.h>
+#include <libavutil/display.h>
 }
 
 #include <android/bitmap.h>
 #include "frame_loader_context.h"
 #include "log.h"
 
-bool read_frame(FrameLoaderContext *frameLoaderContext, AVPacket *packet, AVFrame *frame, AVCodecContext *videoCodecContext) {
+bool read_frame(FrameLoaderContext *frameLoaderContext, AVPacket *packet, AVFrame *frame,
+                AVCodecContext *videoCodecContext) {
     while (av_read_frame(frameLoaderContext->avFormatContext, packet) >= 0) {
         if (packet->stream_index != frameLoaderContext->videoStreamIndex) {
             continue;
@@ -33,7 +35,8 @@ bool read_frame(FrameLoaderContext *frameLoaderContext, AVPacket *packet, AVFram
 }
 
 
-bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle, int64_t time_millis, jobject jBitmap) {
+bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle, int64_t time_millis,
+                                jobject jBitmap) {
     AndroidBitmapInfo bitmapMetricInfo;
     AndroidBitmap_getInfo(env, jBitmap, &bitmapMetricInfo);
 
@@ -67,7 +70,8 @@ bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle,
     int64_t videoDuration = avVideoStream->duration;
     // In some cases the duration is of a video stream is set to Long.MIN_VALUE and we need compute it in another way
     if (videoDuration == LONG_LONG_MIN && avVideoStream->time_base.den != 0) {
-        videoDuration = frameLoaderContext->avFormatContext->duration / avVideoStream->time_base.den;
+        videoDuration =
+                frameLoaderContext->avFormatContext->duration / avVideoStream->time_base.den;
     }
 
 
@@ -141,6 +145,140 @@ bool frame_extractor_load_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle,
     return resultValue;
 }
 
+jobject frame_extractor_get_frame(JNIEnv *env, int64_t jFrameLoaderContextHandle, int64_t time_millis) {
+    auto *frameLoaderContext = frame_loader_context_from_handle(jFrameLoaderContextHandle);
+    if (!frameLoaderContext || !frameLoaderContext->avFormatContext ||
+        !frameLoaderContext->parameters) {
+        return nullptr;
+    }
+
+    auto pixelFormat = static_cast<AVPixelFormat>(frameLoaderContext->parameters->format);
+    if (pixelFormat == AV_PIX_FMT_NONE) {
+        return nullptr;
+    }
+
+    AVStream *avVideoStream = frameLoaderContext->avFormatContext->streams[frameLoaderContext->videoStreamIndex];
+    if (!avVideoStream) {
+        return nullptr;
+    }
+
+    int srcW = frameLoaderContext->parameters->width;
+    int srcH = frameLoaderContext->parameters->height;
+
+    // Determine bitmap dimensions based on rotation
+    int bitmapWidth = srcW > 0 ? srcW : 1920;
+    int bitmapHeight = srcH > 0 ? srcH : 1080;
+
+    // Create Java Bitmap
+    jclass bitmapClass = env->FindClass("android/graphics/Bitmap");
+    jmethodID createBitmapMethod = env->GetStaticMethodID(bitmapClass, "createBitmap",
+                                                          "(IILandroid/graphics/Bitmap$Config;)Landroid/graphics/Bitmap;");
+    jclass bitmapConfigClass = env->FindClass("android/graphics/Bitmap$Config");
+    jfieldID argb8888FieldID = env->GetStaticFieldID(bitmapConfigClass, "ARGB_8888",
+                                                     "Landroid/graphics/Bitmap$Config;");
+    jobject argb8888Obj = env->GetStaticObjectField(bitmapConfigClass, argb8888FieldID);
+    jobject jBitmap = env->CallStaticObjectMethod(bitmapClass, createBitmapMethod, bitmapWidth,
+                                                  bitmapHeight, argb8888Obj);
+
+    SwsContext *scalingContext = sws_getContext(
+            srcW, srcH, pixelFormat,
+            bitmapWidth, bitmapHeight, AV_PIX_FMT_RGBA,
+            SWS_BICUBIC, nullptr, nullptr, nullptr);
+
+    if (!scalingContext) {
+        return nullptr;
+    }
+
+    int64_t videoDuration = avVideoStream->duration;
+    if (videoDuration == LONG_LONG_MIN && avVideoStream->time_base.den != 0) {
+        videoDuration = av_rescale_q(frameLoaderContext->avFormatContext->duration, AV_TIME_BASE_Q,
+                                     avVideoStream->time_base);
+    }
+
+    int64_t seekPosition = (time_millis != -1) ?
+                           av_rescale_q(time_millis, AV_TIME_BASE_Q, avVideoStream->time_base) :
+                           videoDuration / 3;
+
+    seekPosition = FFMIN(seekPosition, videoDuration);
+
+    AVPacket *packet = av_packet_alloc();
+    AVFrame *frame = av_frame_alloc();
+    if (!packet || !frame) {
+        sws_freeContext(scalingContext);
+        av_packet_free(&packet);
+        av_frame_free(&frame);
+        return nullptr;
+    }
+
+    AVCodecContext *videoCodecContext = avcodec_alloc_context3(frameLoaderContext->avVideoCodec);
+    if (!videoCodecContext ||
+        avcodec_parameters_to_context(videoCodecContext, frameLoaderContext->parameters) < 0 ||
+        avcodec_open2(videoCodecContext, frameLoaderContext->avVideoCodec, nullptr) < 0) {
+        sws_freeContext(scalingContext);
+        av_packet_free(&packet);
+        av_frame_free(&frame);
+        avcodec_free_context(&videoCodecContext);
+        return nullptr;
+    }
+
+    av_seek_frame(frameLoaderContext->avFormatContext,
+                  frameLoaderContext->videoStreamIndex,
+                  seekPosition,
+                  AVSEEK_FLAG_BACKWARD);
+
+    bool resultValue = read_frame(frameLoaderContext, packet, frame, videoCodecContext);
+
+    if (!resultValue) {
+        av_seek_frame(frameLoaderContext->avFormatContext,
+                      frameLoaderContext->videoStreamIndex,
+                      0,
+                      0);
+        resultValue = read_frame(frameLoaderContext, packet, frame, videoCodecContext);
+    }
+
+    if (resultValue) {
+        void *bitmapBuffer;
+        if (AndroidBitmap_lockPixels(env, jBitmap, &bitmapBuffer) < 0) {
+            resultValue = false;
+        } else {
+            AVFrame *frameForDrawing = av_frame_alloc();
+            if (frameForDrawing) {
+                av_image_fill_arrays(frameForDrawing->data,
+                                     frameForDrawing->linesize,
+                                     static_cast<const uint8_t *>(bitmapBuffer),
+                                     AV_PIX_FMT_RGBA,
+                                     bitmapWidth,
+                                     bitmapHeight,
+                                     1);
+                sws_scale(scalingContext,
+                          frame->data,
+                          frame->linesize,
+                          0,
+                          frame->height,
+                          frameForDrawing->data,
+                          frameForDrawing->linesize);
+
+                av_frame_free(&frameForDrawing);
+            }
+            AndroidBitmap_unlockPixels(env, jBitmap);
+        }
+    }
+
+    av_packet_free(&packet);
+    av_frame_free(&frame);
+    avcodec_free_context(&videoCodecContext);
+    sws_freeContext(scalingContext);
+
+    if (resultValue) {
+        return jBitmap;
+    } else {
+        env->DeleteLocalRef(jBitmap);
+        return nullptr;
+    }
+
+}
+
+
 extern "C"
 JNIEXPORT void JNICALL
 Java_io_github_anilbeesetti_nextlib_mediainfo_FrameLoader_nativeRelease(JNIEnv *env, jclass clazz,
@@ -153,6 +291,14 @@ Java_io_github_anilbeesetti_nextlib_mediainfo_FrameLoader_nativeLoadFrame(JNIEnv
                                                                           jlong jFrameLoaderContextHandle,
                                                                           jlong time_millis,
                                                                           jobject jBitmap) {
-    bool successfullyLoaded = frame_extractor_load_frame(env, jFrameLoaderContextHandle, time_millis, jBitmap);
+    bool successfullyLoaded = frame_extractor_load_frame(env, jFrameLoaderContextHandle,
+                                                         time_millis, jBitmap);
     return static_cast<jboolean>(successfullyLoaded);
+}
+extern "C"
+JNIEXPORT jobject JNICALL
+Java_io_github_anilbeesetti_nextlib_mediainfo_FrameLoader_nativeGetFrame(JNIEnv *env, jclass clazz,
+                                                                         jlong jFrameLoaderContextHandle,
+                                                                         jlong time_millis) {
+    return frame_extractor_get_frame(env, jFrameLoaderContextHandle, time_millis);
 }

mediainfo/src/main/cpp/mediainfo.cpp

@@ -5,8 +5,9 @@
 #include "frame_loader_context.h"
 
 extern "C" {
-#include "libavformat/avformat.h"
-#include "libavcodec/codec_desc.h"
+#include <libavformat/avformat.h>
+#include <libavcodec/codec_desc.h>
+#include <libavutil/display.h>
 }
 
 static char *get_string(AVDictionary *metadata, const char *key) {
@@ -43,7 +44,8 @@ void onMediaInfoFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *a
                                     duration_ms);
 }
 
-void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
+void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
+                        int index) {
     AVStream *stream = avFormatContext->streams[index];
     AVCodecParameters *parameters = stream->codecpar;
 
@@ -61,9 +63,12 @@ void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
     }
 
 
-    AVRational guessedFrameRate = av_guess_frame_rate(avFormatContext, avFormatContext->streams[index], nullptr);
+    AVRational guessedFrameRate = av_guess_frame_rate(avFormatContext,
+                                                      avFormatContext->streams[index],
+                                                      nullptr);
 
-    double resultFrameRate = guessedFrameRate.den == 0 ? 0.0 : guessedFrameRate.num / (double) guessedFrameRate.den;
+    double resultFrameRate =
+            guessedFrameRate.den == 0 ? 0.0 : guessedFrameRate.num / (double) guessedFrameRate.den;
 
     jstring jTitle = env->NewStringUTF(get_title(stream->metadata));
     jstring jCodecName;
@@ -74,6 +79,22 @@ void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
     }
     jstring jLanguage = env->NewStringUTF(get_language(stream->metadata));
 
+    int rotation = 0;
+    AVDictionaryEntry *rotateTag = av_dict_get(stream->metadata, "rotate", nullptr, 0);
+    if (rotateTag && *rotateTag->value) {
+        rotation = atoi(rotateTag->value);
+        rotation %= 360;
+        if (rotation < 0) rotation += 360;
+    }
+    uint8_t *displaymatrix = av_stream_get_side_data(stream,
+                                                     AV_PKT_DATA_DISPLAYMATRIX,
+                                                     nullptr);
+    if (displaymatrix) {
+        double theta = av_display_rotation_get((int32_t *) displaymatrix);
+        rotation = (int) (-theta) % 360;
+        if (rotation < 0) rotation += 360;
+    }
+
     utils_call_instance_method_void(env,
                                     jMediaInfoBuilder,
                                     fields.MediaInfoBuilder.onVideoStreamFoundID,
@@ -86,10 +107,12 @@ void onVideoStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
                                     resultFrameRate,
                                     parameters->width,
                                     parameters->height,
+                                    rotation,
                                     frameLoaderContextHandle);
 }
 
-void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
+void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
+                        int index) {
     AVStream *stream = avFormatContext->streams[index];
     AVCodecParameters *parameters = stream->codecpar;
 
@@ -98,7 +121,8 @@ void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
     auto avSampleFormat = static_cast<AVSampleFormat>(parameters->format);
     auto jSampleFormat = env->NewStringUTF(av_get_sample_fmt_name(avSampleFormat));
     char chLayoutDescription[128];
-    av_channel_layout_describe(&parameters->ch_layout, chLayoutDescription, sizeof(chLayoutDescription));
+    av_channel_layout_describe(&parameters->ch_layout, chLayoutDescription,
+                               sizeof(chLayoutDescription));
 
     jstring jTitle = env->NewStringUTF(get_title(stream->metadata));
     jstring jCodecName;
@@ -125,7 +149,8 @@ void onAudioStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext
                                     jChannelLayout);
 }
 
-void onSubtitleStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
+void onSubtitleStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
+                           int index) {
     AVStream *stream = avFormatContext->streams[index];
     AVCodecParameters *parameters = stream->codecpar;
 
@@ -150,11 +175,12 @@ void onSubtitleStreamFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatConte
                                     stream->disposition);
 }
 
-void onChapterFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext, int index) {
+void onChapterFound(JNIEnv *env, jobject jMediaInfoBuilder, AVFormatContext *avFormatContext,
+                    int index) {
     AVChapter *chapter = avFormatContext->chapters[index];
 
     jstring jTitle = env->NewStringUTF(get_title(chapter->metadata));
-    double time_base =  av_q2d(chapter->time_base);
+    double time_base = av_q2d(chapter->time_base);
     long start_ms = (long) (chapter->start * time_base * 1000.0);
     long end_ms = (long) (chapter->end * time_base * 1000.0);
 
@@ -189,7 +215,7 @@ void media_info_build(JNIEnv *env, jobject jMediaInfoBuilder, const char *uri) {
         AVMediaType type = parameters->codec_type;
         switch (type) {
             case AVMEDIA_TYPE_VIDEO:
-                 onVideoStreamFound(env, jMediaInfoBuilder, avFormatContext, pos);
+                onVideoStreamFound(env, jMediaInfoBuilder, avFormatContext, pos);
                 break;
             case AVMEDIA_TYPE_AUDIO:
                 onAudioStreamFound(env, jMediaInfoBuilder, avFormatContext, pos);
@@ -207,7 +233,9 @@ void media_info_build(JNIEnv *env, jobject jMediaInfoBuilder, const char *uri) {
 
 extern "C"
 JNIEXPORT void JNICALL
-Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromFD(JNIEnv *env, jobject thiz, jint file_descriptor) {
+Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromFD(JNIEnv *env,
+                                                                                  jobject thiz,
+                                                                                  jint file_descriptor) {
     char pipe[32];
     sprintf(pipe, "pipe:%d", file_descriptor);
 
@@ -216,7 +244,9 @@ Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromF
 
 extern "C"
 JNIEXPORT void JNICALL
-Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromPath(JNIEnv *env, jobject thiz, jstring jFilePath) {
+Java_io_github_anilbeesetti_nextlib_mediainfo_MediaInfoBuilder_nativeCreateFromPath(JNIEnv *env,
+                                                                                    jobject thiz,
+                                                                                    jstring jFilePath) {
     const char *cFilePath = env->GetStringUTFChars(jFilePath, nullptr);
 
     media_info_build(env, thiz, cFilePath);

mediainfo/src/main/cpp/utils.cpp

@@ -61,7 +61,7 @@ int utils_fields_init(JavaVM *vm) {
     GET_ID(GetMethodID,
            fields.MediaInfoBuilder.onVideoStreamFoundID,
            fields.MediaInfoBuilder.clazz,
-           "onVideoStreamFound", "(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;IJDIIJ)V"
+           "onVideoStreamFound", "(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;IJDIIIJ)V"
     );
 
     GET_ID(GetMethodID,

mediainfo/src/main/java/io/github/anilbeesetti/nextlib/mediainfo/FrameLoader.kt

@@ -9,6 +9,11 @@ class FrameLoader internal constructor(private var frameLoaderContextHandle: Lon
         return nativeLoadFrame(frameLoaderContextHandle, durationMillis, bitmap)
     }
 
+    fun getFrame(durationMillis: Long): Bitmap? {
+        require(frameLoaderContextHandle != -1L)
+        return nativeGetFrame(frameLoaderContextHandle, durationMillis)
+    }
+
     fun release() {
         nativeRelease(frameLoaderContextHandle)
         frameLoaderContextHandle = -1
@@ -20,5 +25,8 @@ class FrameLoader internal constructor(private var frameLoaderContextHandle: Lon
 
         @JvmStatic
         private external fun nativeLoadFrame(handle: Long, durationMillis: Long, bitmap: Bitmap): Boolean
+
+        @JvmStatic
+        private external fun nativeGetFrame(handle: Long, durationMillis: Long): Bitmap?
     }
 }

mediainfo/src/main/java/io/github/anilbeesetti/nextlib/mediainfo/MediaInfo.kt

@@ -27,11 +27,29 @@ data class MediaInfo(
         if (videoStream == null) return null
         val bitmap = Bitmap.createBitmap(videoStream.frameWidth.takeIf { it > 0 } ?: 1920, videoStream.frameHeight.takeIf { it > 0 } ?: 1080, Bitmap.Config.ARGB_8888)
         val result = frameLoader?.loadFrameInto(bitmap, durationMillis)
-        return if (result == true) bitmap else null
+        return if (result == true) bitmap.rotate(videoStream.rotation) else null
+    }
+
+    /**
+     * Retrieves a video frame as a Bitmap at a specific duration in milliseconds from the video stream.
+     *
+     * @param durationMillis The timestamp in milliseconds at which to retrieve the video frame.
+     *                       If set to -1, the frame will be retrieved at one-third of the video's duration.
+     * @return A Bitmap containing the video frame if retrieval is successful, or null if an error occurs.
+     */
+    fun getFrameAt(durationMillis: Long = -1): Bitmap? {
+        if (videoStream == null) return null
+        return frameLoader?.getFrame(durationMillis)?.rotate(videoStream.rotation)
     }
 
     fun release() {
         frameLoader?.release()
         frameLoader = null
     }
 }
+
+private fun Bitmap.rotate(degrees: Int): Bitmap {
+    val matrix = android.graphics.Matrix()
+    matrix.postRotate(degrees.toFloat())
+    return Bitmap.createBitmap(this, 0, 0, width, height, matrix, true)
+}