VideoPicture.cpp

/*******************************************************************************
 * Copyright (c) 2024, 2026, Olivier Ayache.  All rights reserved.
 *
 * This file is part of AVPKit.
 *
 * AVPKit is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * AVPKit is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with AVPKit.  If not, see <http://www.gnu.org/licenses/>.
 *******************************************************************************/
 
#include <stdexcept>
// for memset
#include <cstring>
#include <com/avpkit/ferry/Logger.h>
#include <com/avpkit/ferry/RefPointer.h>
#include <com/avpkit/core/Global.h>
#include "com/avpkit/core/VideoPicture.h"
 
extern "C" {
#include <libavcodec/mediacodec.h> 
#ifdef __APPLE__
#include <CoreMedia/CMSampleBuffer.h>
#endif
}
 
VS_LOG_SETUP(VS_CPP_PACKAGE);
 
namespace com { namespace avpkit { namespace core
{
 
  VideoPicture :: VideoPicture()
  {
    mIsComplete = false;
    mFrame = av_frame_alloc();
    if (!mFrame)
      throw std::bad_alloc();
    // Set the private data pointer to point to me.
    mFrame->opaque = this;
    mFrame->width = -1;
    mFrame->height = -1;
 
    mFrame->format = (int) IPixelFormat::NONE;
    mTimeBase = IRational::make(1, 1000000);
 
#ifdef __APPLE__
    mCMSampleBuffer = NULL;
#endif
    
  }
 
  VideoPicture :: ~VideoPicture()
  {
    if (mFrame)
      av_free(mFrame);
    mFrame = 0;
    mBuffer = 0;
    mTimeBase = 0;
#ifdef __APPLE__
    if (mCMSampleBuffer){
        CFRelease(mCMSampleBuffer);
    }
#endif
  }
 
  VideoPicture*
  VideoPicture :: make(IPixelFormat::Type format, int width, int height)
  {
    VideoPicture * retval=0;
    try {
      retval = VideoPicture::make();
      retval->mFrame->format = format;
      retval->mFrame->width = width;
      retval->mFrame->height = height;
      // default new frames to be key frames
      retval->setKeyFrame(true);
    }
    catch (std::bad_alloc &e)
    {
      VS_REF_RELEASE(retval);
      throw e;
    }
    catch (std::exception& e)
    {
      VS_LOG_DEBUG("error: %s", e.what());
      VS_REF_RELEASE(retval);
    }
 
    return retval;
  }
  
  VideoPicture*
  VideoPicture :: make(
      com::avpkit::ferry::IBuffer* buffer, IPixelFormat::Type format,
      int width, int height)
  {
    if (!buffer)
      return 0;
    VideoPicture *retval = 0;
    try {
      retval = make(format, width,height);
      if (!retval)
        throw std::bad_alloc();
      
      int32_t size = retval->getSize();
      if (size > 0 && size > buffer->getBufferSize())
        throw std::runtime_error("input buffer is not large enough for given picture");
      
      retval->mBuffer.reset(buffer, true);
      unsigned char* bytes = (unsigned char*)buffer->getBytes(0, size);
      if (!bytes)
        throw std::runtime_error("could not access raw memory in buffer");
 
      (void) av_image_fill_arrays(retval->mFrame->data,
              retval->mFrame->linesize,
              bytes, 
              (AVPixelFormat) format, 
              width, 
              height, 
              1);
 
    }
    catch (std::bad_alloc &e)
    {
      VS_REF_RELEASE(retval);
      throw e;
    }
    catch (std::exception& e)
    {
      VS_LOG_DEBUG("error: %s", e.what());
      VS_REF_RELEASE(retval);
    }
    return retval;
  }
 
  void
  VideoPicture :: setData(com::avpkit::ferry::IBuffer* buffer)
  {
    if (!buffer) return;
    mBuffer.reset(buffer, true);
  }
  
  bool
  VideoPicture :: copy(IVideoPicture * srcFrame)
  {
    bool result = false;
    try
    {
      if (!srcFrame)
        throw std::runtime_error("empty source frame to copy");
 
      if (!srcFrame->isComplete())
        throw std::runtime_error("source frame is not complete");
 
      VideoPicture* src = static_cast<VideoPicture*>(srcFrame);
      if (!src)
        throw std::runtime_error("src frame is not of right subtype");
 
      // now copy the data
      allocInternalFrameBuffer();
 
      // get the raw buffers
      unsigned char* srcBuffer = (unsigned char*)src->mBuffer->getBytes(0, src->getSize());
      unsigned char* dstBuffer = (unsigned char*)mBuffer->getBytes(0, getSize());
      if (!srcBuffer || !dstBuffer)
        throw std::runtime_error("could not get buffer to copy");
      memcpy(dstBuffer, srcBuffer, getSize());
 
      this->setComplete(true,
          srcFrame->getPixelType(),
          srcFrame->getWidth(),
          srcFrame->getHeight(),
          srcFrame->getPts());
      result = true;
    }
    catch (std::exception & e)
    {
      VS_LOG_DEBUG("error: %s", e.what());
      result = false;
    }
    return result;
  }
 
  com::avpkit::ferry::IBuffer*
  VideoPicture :: getData()
  {
    com::avpkit::ferry::IBuffer *retval = 0;
    try {
      if (getSize() > 0) {
        if (!mBuffer || mBuffer->getBufferSize() < getSize())
        {
          allocInternalFrameBuffer();
        }
        retval = mBuffer.get();
        if (!retval) {
          throw std::bad_alloc();
        }
      }
    } catch (std::bad_alloc &e) {
      VS_REF_RELEASE(retval);
      throw e;
    } catch (std::exception & e)
    {
      VS_LOG_DEBUG("Error: %s", e.what());
      VS_REF_RELEASE(retval);
    }
    return retval;
  }
              
  void 
  VideoPicture::render(bool drop, int64_t timeStamp) {
      if (mFrame && (av_pix_fmt_desc_get((AVPixelFormat) mFrame->format)->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
        #ifdef __ANDROID__
          mediacodec_render_frame(mFrame, timeStamp, drop);
        #elif __APPLE__
          CVImageBufferRef image = (CVImageBufferRef) mFrame->data[3];
          CMVideoFormatDescriptionRef formatDescription;
          if (mCMSampleBuffer){
              CFRelease(mCMSampleBuffer);
          }
          CMTime duration = kCMTimeIndefinite;
          CMTime pts = CMTimeMake(timeStamp , 1000000000);
          const CMSampleTimingInfo timingInfo = {
                        .decodeTimeStamp = kCMTimeInvalid,
                        .duration = duration,
                        .presentationTimeStamp = pts
                    };
          CMVideoFormatDescriptionCreateForImageBuffer(NULL, image, &formatDescription);
          CMSampleBufferCreateReadyWithImageBuffer(NULL, image, formatDescription, &timingInfo, &mCMSampleBuffer);
          return sampleBuffer;
        #endif
      }
  }
 
  void*
  VideoPicture :: getOpaqueData()
  {
    #ifdef __APPLE__
      return mCMSampleBuffer;
    #endif
 
    return NULL;
  }
  
  void
  VideoPicture :: fillAVFrame(AVFrame *frame)
  {
    if (!mBuffer || mBuffer->getBufferSize() < getSize())
      allocInternalFrameBuffer();
    unsigned char* buffer = (unsigned char*)mBuffer->getBytes(0, getSize());
    // This is an inherently unsafe operation; it copies over all the bits in the AVFrame
    memcpy(frame, mFrame, sizeof(AVFrame));
    //*frame = *mFrame;
    // and then relies on avpicture_fill to overwrite any areas in frame that
    // are pointed to the wrong place.
    av_image_fill_arrays(frame->data,
              frame->linesize,
              buffer, 
              (AVPixelFormat) frame->format, 
              frame->width, 
              frame->height, 
              1);
    frame->quality = getQuality();
//    frame->type = FF_BUFFER_TYPE_USER;
  }
 
  void
  VideoPicture :: copyAVFrame(AVFrame* frame, IPixelFormat::Type pixel,
      int32_t width, int32_t height)
  {
    try
    {
      // Need to copy the contents of frame->data to our
      // internal buffer.
      VS_ASSERT(frame, "no frame?");
      if (!(av_pix_fmt_desc_get((AVPixelFormat) pixel)->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
        VS_ASSERT(frame->data[0], "no data in frame");
        // resize the frame to the AVFrame
        mFrame->width = width;
        mFrame->height = height;
        mFrame->format = (int)pixel;
      
        int bufSize = getSize();
        if (bufSize <= 0)
          throw std::runtime_error("invalid size for frame");
 
        if (!mBuffer || mBuffer->getBufferSize() < bufSize)
          // reuse buffers if we can.
          allocInternalFrameBuffer();
 
        uint8_t* buffer = (uint8_t*)mBuffer->getBytes(0, bufSize);
        if (!buffer)
          throw std::runtime_error("really?  no buffer");
 
        if (frame->data[0])
        {
          // Make sure the frame isn't already using our buffer
          if(buffer != frame->data[0])
          {
            av_image_fill_arrays(mFrame->data, mFrame->linesize, buffer, 
              (AVPixelFormat) frame->format, width, height, 1);
            av_image_copy(mFrame->data, mFrame->linesize, (const uint8_t **)frame->data,
                  frame->linesize, (AVPixelFormat)frame->format, frame->width, frame->height);
          }
          mFrame->key_frame = frame->key_frame;
        }
        else
        {
          throw std::runtime_error("no data in frame to copy");
        }
      } 
      else 
      {
        mFrame->width = width;
        mFrame->height = height;
//      mFrame->format = (int) pixel;
        if (pixel != IPixelFormat::MEDIACODEC) 
        {
            //mFrame->hw_frames_ctx = frame->hw_frames_ctx;
          av_hwframe_transfer_data(mFrame, frame, 0);
        } 
        else 
        {
            //av_frame_copy_props(mFrame, frame);
          mFrame->data[3] = frame->data[3];    
        }
      }
    }
    catch (std::exception & e)
    {
      VS_LOG_DEBUG("error: %s", e.what());
    }
  }
 
  AVFrame*
  VideoPicture :: getAVFrame()
  {
    if (!mBuffer || mBuffer->getBufferSize() < getSize())
    {
      // reuse buffers if we can.
      allocInternalFrameBuffer();
    }
    return mFrame;
  }
  
  int
  VideoPicture :: getDataLineSize(int lineNo)
  {
    int retval = -1;
    if (getAVFrame()
        && lineNo >= 0
        && (unsigned int) lineNo < (sizeof(mFrame->linesize)/sizeof(mFrame->linesize[0])))
      retval = mFrame->linesize[lineNo];
    return retval;
  }
 
  bool
  VideoPicture :: isKeyFrame()
  {
    return (mFrame ? mFrame->key_frame : false);
  }
 
  void
  VideoPicture :: setKeyFrame(bool aIsKey)
  {
    if (mFrame)
      mFrame->key_frame = aIsKey;
  }
  
  int64_t
  VideoPicture :: getPts()
  {
    return (mFrame ? mFrame->pts : -1);
  }
 
  void
  VideoPicture :: setPts(int64_t value)
  {
    if (mFrame)
      mFrame->pts = value;
  }
 
  int
  VideoPicture :: getQuality()
  {
    return (mFrame ? mFrame->quality : FF_LAMBDA_MAX);
  }
 
  void
  VideoPicture :: setQuality(int newQuality)
  {
    if (newQuality < 0 || newQuality > FF_LAMBDA_MAX)
      newQuality = FF_LAMBDA_MAX;
    if (mFrame)
      mFrame->quality = newQuality;
  }
 
  void
  VideoPicture :: setComplete(
      bool aIsComplete,
      IPixelFormat::Type format,
      int width,
      int height,
      int64_t pts
  )
  {
    try {
      mIsComplete = aIsComplete;
 
      if (mIsComplete)
      {
        setPts(pts);
      }
 
      if (!mFrame)
        throw std::runtime_error("no AVFrame allocated");
      if (format != IPixelFormat::NONE && mFrame->format != (int)IPixelFormat::NONE && (int)format != mFrame->format)
        throw std::runtime_error("pixel formats don't match");
      if (width > 0 && mFrame->width >0 && width != mFrame->width)
        throw std::runtime_error("width does not match");
      if (height > 0 && mFrame->height > 0 && height != mFrame->height)
        throw std::runtime_error("height does not match");
    }
    catch (std::exception& e)
    {
      VS_LOG_DEBUG("error: %s", e.what());
    }
  }
 
  int32_t
  VideoPicture :: getSize()
  {
    int retval = -1;
    if (mFrame->width > 0 && mFrame->height > 0)
      retval = av_image_get_buffer_size((AVPixelFormat)mFrame->format, mFrame->width, mFrame->height, 1);
    return retval;
  }
 
  void
  VideoPicture :: allocInternalFrameBuffer()
  {
    int bufSize = getSize();
    if (bufSize <= 0)
      throw std::runtime_error("invalid size for frame");
 
    // reuse buffers if we can.
    if (!mBuffer || mBuffer->getBufferSize() < bufSize)
    {
      // Now, it turns out some accelerated assembly functions will
      // read at least a word past the end of an image buffer, so
      // we make space for that to happen.
      // I arbitrarily choose the sizeof a long-long (64 bit).
 
      // note that if the user has passed in their own buffer that is
      // the right 'size' but doesnt have this padding, we let it through anyway.
      int extraBytes=sizeof(int64_t);
 
      // Make our copy buffer.
      mBuffer = com::avpkit::ferry::IBuffer::make(this, bufSize+extraBytes);
      if (!mBuffer) {
        throw std::bad_alloc();
      }
 
      // Now, to further work around issues, I added the extra 8-bytes,
      // and now I'm going to zero just those 8-bytes out.  I don't
      // zero-out the whole buffer because I want Valgrind to detect
      // if it's not written to first.  But I know this overrun
      // issue exists in the MMX conversions in SWScale for some libraries,
      // so I'm going to fake it out here.
      {
        unsigned char * buf =
          ((unsigned char*)mBuffer->getBytes(0, bufSize+extraBytes));
 
        memset(buf+bufSize, 0, extraBytes);
      }
    }
    uint8_t* buffer = (uint8_t*)mBuffer->getBytes(0, bufSize);
    if (!buffer)
      throw std::bad_alloc();
 
    int imageSize = av_image_fill_arrays(mFrame->data,
        mFrame->linesize,
        buffer,
        (AVPixelFormat)mFrame->format,
        mFrame->width,
        mFrame->height,
        1);
    if (imageSize != bufSize)
      throw std::runtime_error("could not fill picture");
 
//    mFrame->type = FF_BUFFER_TYPE_USER;
    VS_ASSERT(mFrame->data[0] != 0, "Empty buffer");
  }
 
  IVideoPicture::PictType
  VideoPicture :: getPictureType()
  {
    IVideoPicture::PictType retval = IVideoPicture::DEFAULT_TYPE;
    if (mFrame)
      retval = (PictType) mFrame->pict_type;
    return retval;
  }
  
  void
  VideoPicture :: setPictureType(IVideoPicture::PictType type)
  {
    if (mFrame)
      mFrame->pict_type = (enum AVPictureType) type;
  }
  
  void
  VideoPicture::setSideData(IVideoPicture::FrameDataType type, com::avpkit::ferry::IBuffer* buffer) {
      if (mFrame){
        av_frame_new_side_data(mFrame, (AVFrameSideDataType)type, buffer->getSize());
      }
  }
 
}}}