#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include "opencv2/core.hpp"
#include <opencv2/core/utility.hpp>
#include "opencv2/video.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/videoio.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/videostab.hpp"
#include "opencv2/opencv_modules.hpp"
#define arg(name) cmd.get<string>(name)
#define argb(name) cmd.get<bool>(name)
#define argi(name) cmd.get<int>(name)
#define argf(name) cmd.get<float>(name)
#define argd(name) cmd.get<double>(name)
using namespace std;
using namespace cv;
using namespace cv::videostab;
Ptr<IFrameSource> stabilizedFrames;
string saveMotionsPath;
double outputFps;
string outputPath;
bool quietMode;
void run();
void saveMotionsIfNecessary();
void printHelp();
MotionModel motionModel(const string &str);
void run()
{
VideoWriter writer;
Mat stabilizedFrame;
int nframes = 0;
// for each stabilized frame
while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty())
{
nframes++;
// init writer (once) and save stabilized frame
if (!outputPath.empty())
{
if (!writer.isOpened())
writer.open(outputPath, VideoWriter::fourcc('X','V','I','D'),
outputFps, stabilizedFrame.size());
writer << stabilizedFrame;
}
// show stabilized frame
if (!quietMode)
{
imshow("stabilizedFrame", stabilizedFrame);
char key = static_cast<char>(waitKey(3));
if (key == 27) { cout << endl; break; }
}
}
cout << "processed frames: " << nframes << endl
<< "finished\n";
}
void printHelp()
{
cout << "OpenCV video stabilizer.\n"
"Usage: videostab <file_path> [arguments]\n\n"
"Arguments:\n"
" -m, --model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n"
" Set motion model. The default is affine.\n"
" -lp, --lin-prog-motion-est=(yes|no)\n"
" Turn on/off LP based motion estimation. The default is no.\n"
" --subset=(<int_number>|auto)\n"
" Number of random samples per one motion hypothesis. The default is auto.\n"
" --thresh=(<float_number>|auto)\n"
" Maximum error to classify match as inlier. The default is auto.\n"
" --outlier-ratio=<float_number>\n"
" Motion estimation outlier ratio hypothesis. The default is 0.5.\n"
" --min-inlier-ratio=<float_number>\n"
" Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n"
" --nkps=<int_number>\n"
" Number of keypoints to find in each frame. The default is 1000.\n"
" --local-outlier-rejection=(yes|no)\n"
" Perform local outlier rejection. The default is no.\n\n"
" -sm, --save-motions=(<file_path>|no)\n"
" Save estimated motions into file. The default is no.\n"
" -lm, --load-motions=(<file_path>|no)\n"
" Load motions from file. The default is no.\n\n"
" -r, --radius=<int_number>\n"
" Set sliding window radius. The default is 15.\n"
" --stdev=(<float_number>|auto)\n"
" Set smoothing weights standard deviation. The default is auto\n"
" (i.e. sqrt(radius)).\n"
" -lps, --lin-prog-stab=(yes|no)\n"
" Turn on/off linear programming based stabilization method.\n"
" --lps-trim-ratio=(<float_number>|auto)\n"
" Trimming ratio used in linear programming based method.\n"
" --lps-w1=(<float_number>|1)\n"
" 1st derivative weight. The default is 1.\n"
" --lps-w2=(<float_number>|10)\n"
" 2nd derivative weight. The default is 10.\n"
" --lps-w3=(<float_number>|100)\n"
" 3rd derivative weight. The default is 100.\n"
" --lps-w4=(<float_number>|100)\n"
" Non-translation motion components weight. The default is 100.\n\n"
" --deblur=(yes|no)\n"
" Do deblurring.\n"
" --deblur-sens=<float_number>\n"
" Set deblurring sensitivity (from 0 to +inf). The default is 0.1.\n\n"
" -t, --trim-ratio=<float_number>\n"
" Set trimming ratio (from 0 to 0.5). The default is 0.1.\n"
" -et, --est-trim=(yes|no)\n"
" Estimate trim ratio automatically. The default is yes.\n"
" -ic, --incl-constr=(yes|no)\n"
" Ensure the inclusion constraint is always satisfied. The default is no.\n\n"
" -bm, --border-mode=(replicate|reflect|const)\n"
" Set border extrapolation mode. The default is replicate.\n\n"
" --mosaic=(yes|no)\n"
" Do consistent mosaicing. The default is no.\n"
" --mosaic-stdev=<float_number>\n"
" Consistent mosaicing stdev threshold. The default is 10.0.\n\n"
" -mi, --motion-inpaint=(yes|no)\n"
" Do motion inpainting (requires CUDA support). The default is no.\n"
" --mi-dist-thresh=<float_number>\n"
" Estimated flow distance threshold for motion inpainting. The default is 5.0.\n\n"
" -ci, --color-inpaint=(no|average|ns|telea)\n"
" Do color inpainting. The defailt is no.\n"
" --ci-radius=<float_number>\n"
" Set color inpainting radius (for ns and telea options only).\n"
" The default is 2.0\n\n"
" -ws, --wobble-suppress=(yes|no)\n"
" Perform wobble suppression. The default is no.\n"
" --ws-lp=(yes|no)\n"
" Turn on/off LP based motion estimation. The default is no.\n"
" --ws-period=<int_number>\n"
" Set wobble suppression period. The default is 30.\n"
" --ws-model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n"
" Set wobble suppression motion model (must have more DOF than motion \n"
" estimation model). The default is homography.\n"
" --ws-subset=(<int_number>|auto)\n"
" Number of random samples per one motion hypothesis. The default is auto.\n"
" --ws-thresh=(<float_number>|auto)\n"
" Maximum error to classify match as inlier. The default is auto.\n"
" --ws-outlier-ratio=<float_number>\n"
" Motion estimation outlier ratio hypothesis. The default is 0.5.\n"
" --ws-min-inlier-ratio=<float_number>\n"
" Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n"
" --ws-nkps=<int_number>\n"
" Number of keypoints to find in each frame. The default is 1000.\n"
" --ws-local-outlier-rejection=(yes|no)\n"
" Perform local outlier rejection. The default is no.\n\n"
" -sm2, --save-motions2=(<file_path>|no)\n"
" Save motions estimated for wobble suppression. The default is no.\n"
" -lm2, --load-motions2=(<file_path>|no)\n"
" Load motions for wobble suppression from file. The default is no.\n\n"
" -gpu=(yes|no)\n"
" Use CUDA optimization whenever possible. The default is no.\n\n"
" -o, --output=(no|<file_path>)\n"
" Set output file path explicitely. The default is stabilized.avi.\n"
" --fps=(<float_number>|auto)\n"
" Set output video FPS explicitely. By default the source FPS is used (auto).\n"
" -q, --quiet\n"
" Don't show output video frames.\n\n"
" -h, --help\n"
" Print help.\n\n"
"Note: some argument configurations lead to two passes, some to single pass.\n\n";
}
// motion estimator builders are for concise creation of motion estimators
class IMotionEstimatorBuilder
{
public:
virtual ~IMotionEstimatorBuilder() {}
virtual Ptr<ImageMotionEstimatorBase> build() = 0;
protected:
IMotionEstimatorBuilder(CommandLineParser &command) : cmd(command) {}
CommandLineParser cmd;
};
class MotionEstimatorRansacL2Builder : public IMotionEstimatorBuilder
{
public:
MotionEstimatorRansacL2Builder(CommandLineParser &command, bool use_gpu, const string &_prefix = "")
: IMotionEstimatorBuilder(command), gpu(use_gpu), prefix(_prefix) {}
virtual Ptr<ImageMotionEstimatorBase> build()
{
Ptr<MotionEstimatorRansacL2> est = makePtr<MotionEstimatorRansacL2>(motionModel(arg(prefix + "model")));
RansacParams ransac = est->ransacParams();
if (arg(prefix + "subset") != "auto")
ransac.size = argi(prefix + "subset");
if (arg(prefix + "thresh") != "auto")
ransac.thresh = argf(prefix + "thresh");
ransac.eps = argf(prefix + "outlier-ratio");
est->setRansacParams(ransac);
est->setMinInlierRatio(argf(prefix + "min-inlier-ratio"));
Ptr<IOutlierRejector> outlierRejector = makePtr<NullOutlierRejector>();
if (arg(prefix + "local-outlier-rejection") == "yes")
{
Ptr<TranslationBasedLocalOutlierRejector> tblor = makePtr<TranslationBasedLocalOutlierRejector>();
RansacParams ransacParams = tblor->ransacParams();
if (arg(prefix + "thresh") != "auto")
ransacParams.thresh = argf(prefix + "thresh");
tblor->setRansacParams(ransacParams);
outlierRejector = tblor;
}
#if defined(HAVE_OPENCV_CUDAIMGPROC) && defined(HAVE_OPENCV_CUDAOPTFLOW)
if (gpu)
{
Ptr<KeypointBasedMotionEstimatorGpu> kbest = makePtr<KeypointBasedMotionEstimatorGpu>(est);
kbest->setOutlierRejector(outlierRejector);
return kbest;
}
#endif
Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est);
kbest->setDetector(GFTTDetector::create(argi(prefix + "nkps")));
kbest->setOutlierRejector(outlierRejector);
return kbest;
}
private:
bool gpu;
string prefix;
};
class MotionEstimatorL1Builder : public IMotionEstimatorBuilder
{
public:
MotionEstimatorL1Builder(CommandLineParser &command, bool use_gpu, const string &_prefix = "")
: IMotionEstimatorBuilder(command), gpu(use_gpu), prefix(_prefix) {}
virtual Ptr<ImageMotionEstimatorBase> build()
{
Ptr<MotionEstimatorL1> est = makePtr<MotionEstimatorL1>(motionModel(arg(prefix + "model")));
Ptr<IOutlierRejector> outlierRejector = makePtr<NullOutlierRejector>();
if (arg(prefix + "local-outlier-rejection") == "yes")
{
Ptr<TranslationBasedLocalOutlierRejector> tblor = makePtr<TranslationBasedLocalOutlierRejector>();
RansacParams ransacParams = tblor->ransacParams();
if (arg(prefix + "thresh") != "auto")
ransacParams.thresh = argf(prefix + "thresh");
tblor->setRansacParams(ransacParams);
outlierRejector = tblor;
}
#if defined(HAVE_OPENCV_CUDAIMGPROC) && defined(HAVE_OPENCV_CUDAOPTFLOW)
if (gpu)
{
Ptr<KeypointBasedMotionEstimatorGpu> kbest = makePtr<KeypointBasedMotionEstimatorGpu>(est);
kbest->setOutlierRejector(outlierRejector);
return kbest;
}
#endif
Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est);
kbest->setDetector(GFTTDetector::create(argi(prefix + "nkps")));
kbest->setOutlierRejector(outlierRejector);
return kbest;
}
private:
bool gpu;
string prefix;
};
int main(int argc, const char **argv)
{
try
{
const char *keys =
"{ @1 | | }"
"{ m model | affine | }"
"{ lp lin-prog-motion-est | no | }"
"{ subset | auto | }"
"{ thresh | auto | }"
"{ outlier-ratio | 0.5 | }"
"{ min-inlier-ratio | 0.1 | }"
"{ nkps | 1000 | }"
"{ extra-kps | 0 | }"
"{ local-outlier-rejection | no | }"
"{ sm save-motions | no | }"
"{ lm load-motions | no | }"
"{ r radius | 15 | }"
"{ stdev | auto | }"
"{ lps lin-prog-stab | no | }"
"{ lps-trim-ratio | auto | }"
"{ lps-w1 | 1 | }"
"{ lps-w2 | 10 | }"
"{ lps-w3 | 100 | }"
"{ lps-w4 | 100 | }"
"{ deblur | no | }"
"{ deblur-sens | 0.1 | }"
"{ et est-trim | yes | }"
"{ t trim-ratio | 0.1 | }"
"{ ic incl-constr | no | }"
"{ bm border-mode | replicate | }"
"{ mosaic | no | }"
"{ ms mosaic-stdev | 10.0 | }"
"{ mi motion-inpaint | no | }"
"{ mi-dist-thresh | 5.0 | }"
"{ ci color-inpaint | no | }"
"{ ci-radius | 2 | }"
"{ ws wobble-suppress | no | }"
"{ ws-period | 30 | }"
"{ ws-model | homography | }"
"{ ws-subset | auto | }"
"{ ws-thresh | auto | }"
"{ ws-outlier-ratio | 0.5 | }"
"{ ws-min-inlier-ratio | 0.1 | }"
"{ ws-nkps | 1000 | }"
"{ ws-extra-kps | 0 | }"
"{ ws-local-outlier-rejection | no | }"
"{ ws-lp | no | }"
"{ sm2 save-motions2 | no | }"
"{ lm2 load-motions2 | no | }"
"{ gpu | no | }"
"{ o output | stabilized.avi | }"
"{ fps | auto | }"
"{ q quiet | | }"
"{ h help | | }";
CommandLineParser cmd(argc, argv, keys);
// parse command arguments
if (argb("help"))
{
printHelp();
return 0;
}
if (arg("gpu") == "yes")
{
cout << "initializing GPU..."; cout.flush();
Mat hostTmp = Mat::zeros(1, 1, CV_32F);
cuda::GpuMat deviceTmp;
deviceTmp.upload(hostTmp);
cout << endl;
}
StabilizerBase *stabilizer = 0;
// check if source video is specified
string inputPath = arg(0);
if (inputPath.empty())
throw runtime_error("specify video file path");
// get source video parameters
Ptr<VideoFileSource> source = makePtr<VideoFileSource>(inputPath);
cout << "frame count (rough): " << source->count() << endl;
if (arg("fps") == "auto")
outputFps = source->fps();
else
outputFps = argd("fps");
// prepare motion estimation builders
Ptr<IMotionEstimatorBuilder> motionEstBuilder;
if (arg("lin-prog-motion-est") == "yes")
motionEstBuilder.reset(new MotionEstimatorL1Builder(cmd, arg("gpu") == "yes"));
else
motionEstBuilder.reset(new MotionEstimatorRansacL2Builder(cmd, arg("gpu") == "yes"));
Ptr<IMotionEstimatorBuilder> wsMotionEstBuilder;
if (arg("ws-lp") == "yes")
wsMotionEstBuilder.reset(new MotionEstimatorL1Builder(cmd, arg("gpu") == "yes", "ws-"));
else
wsMotionEstBuilder.reset(new MotionEstimatorRansacL2Builder(cmd, arg("gpu") == "yes", "ws-"));
// determine whether we must use one pass or two pass stabilizer
bool isTwoPass =
arg("est-trim") == "yes" || arg("wobble-suppress") == "yes" || arg("lin-prog-stab") == "yes";
if (isTwoPass)
{
// we must use two pass stabilizer
TwoPassStabilizer *twoPassStabilizer = new TwoPassStabilizer();
stabilizer = twoPassStabilizer;
twoPassStabilizer->setEstimateTrimRatio(arg("est-trim") == "yes");
// determine stabilization technique
if (arg("lin-prog-stab") == "yes")
{
Ptr<LpMotionStabilizer> stab = makePtr<LpMotionStabilizer>();
stab->setFrameSize(Size(source->width(), source->height()));
stab->setTrimRatio(arg("lps-trim-ratio") == "auto" ? argf("trim-ratio") : argf("lps-trim-ratio"));
stab->setWeight1(argf("lps-w1"));
stab->setWeight2(argf("lps-w2"));
stab->setWeight3(argf("lps-w3"));
stab->setWeight4(argf("lps-w4"));
twoPassStabilizer->setMotionStabilizer(stab);
}
else if (arg("stdev") == "auto")
twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(argi("radius")));
else
twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(argi("radius"), argf("stdev")));
// init wobble suppressor if necessary
if (arg("wobble-suppress") == "yes")
{
Ptr<MoreAccurateMotionWobbleSuppressorBase> ws = makePtr<MoreAccurateMotionWobbleSuppressor>();
if (arg("gpu") == "yes")
#ifdef HAVE_OPENCV_CUDAWARPING
ws = makePtr<MoreAccurateMotionWobbleSuppressorGpu>();
#else
throw runtime_error("OpenCV is built without CUDA support");
#endif
ws->setMotionEstimator(wsMotionEstBuilder->build());
ws->setPeriod(argi("ws-period"));
twoPassStabilizer->setWobbleSuppressor(ws);
MotionModel model = ws->motionEstimator()->motionModel();
if (arg("load-motions2") != "no")
{
ws->setMotionEstimator(makePtr<FromFileMotionReader>(arg("load-motions2")));
ws->motionEstimator()->setMotionModel(model);
}
if (arg("save-motions2") != "no")
{
ws->setMotionEstimator(makePtr<ToFileMotionWriter>(arg("save-motions2"), ws->motionEstimator()));
ws->motionEstimator()->setMotionModel(model);
}
}
}
else
{
// we must use one pass stabilizer
OnePassStabilizer *onePassStabilizer = new OnePassStabilizer();
stabilizer = onePassStabilizer;
if (arg("stdev") == "auto")
onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(argi("radius")));
else
onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(argi("radius"), argf("stdev")));
}
stabilizer->setFrameSource(source);
stabilizer->setMotionEstimator(motionEstBuilder->build());
// cast stabilizer to simple frame source interface to read stabilized frames
stabilizedFrames.reset(dynamic_cast<IFrameSource*>(stabilizer));
MotionModel model = stabilizer->motionEstimator()->motionModel();
if (arg("load-motions") != "no")
{
stabilizer->setMotionEstimator(makePtr<FromFileMotionReader>(arg("load-motions")));
stabilizer->motionEstimator()->setMotionModel(model);
}
if (arg("save-motions") != "no")
{
stabilizer->setMotionEstimator(makePtr<ToFileMotionWriter>(arg("save-motions"), stabilizer->motionEstimator()));
stabilizer->motionEstimator()->setMotionModel(model);
}
stabilizer->setRadius(argi("radius"));
// init deblurer
if (arg("deblur") == "yes")
{
Ptr<WeightingDeblurer> deblurer = makePtr<WeightingDeblurer>();
deblurer->setRadius(argi("radius"));
deblurer->setSensitivity(argf("deblur-sens"));
stabilizer->setDeblurer(deblurer);
}
// set up trimming paramters
stabilizer->setTrimRatio(argf("trim-ratio"));
stabilizer->setCorrectionForInclusion(arg("incl-constr") == "yes");
if (arg("border-mode") == "reflect")
stabilizer->setBorderMode(BORDER_REFLECT);
else if (arg("border-mode") == "replicate")
stabilizer->setBorderMode(BORDER_REPLICATE);
else if (arg("border-mode") == "const")
stabilizer->setBorderMode(BORDER_CONSTANT);
else
throw runtime_error("unknown border extrapolation mode: "
+ cmd.get<string>("border-mode"));
// init inpainter
InpaintingPipeline *inpainters = new InpaintingPipeline();
Ptr<InpainterBase> inpainters_(inpainters);
if (arg("mosaic") == "yes")
{
Ptr<ConsistentMosaicInpainter> inp = makePtr<ConsistentMosaicInpainter>();
inp->setStdevThresh(argf("mosaic-stdev"));
inpainters->pushBack(inp);
}
if (arg("motion-inpaint") == "yes")
{
Ptr<MotionInpainter> inp = makePtr<MotionInpainter>();
inp->setDistThreshold(argf("mi-dist-thresh"));
inpainters->pushBack(inp);
}
if (arg("color-inpaint") == "average")
inpainters->pushBack(makePtr<ColorAverageInpainter>());
else if (arg("color-inpaint") == "ns")
inpainters->pushBack(makePtr<ColorInpainter>(int(INPAINT_NS), argd("ci-radius")));
else if (arg("color-inpaint") == "telea")
inpainters->pushBack(makePtr<ColorInpainter>(int(INPAINT_TELEA), argd("ci-radius")));
else if (arg("color-inpaint") != "no")
throw runtime_error("unknown color inpainting method: " + arg("color-inpaint"));
if (!inpainters->empty())
{
inpainters->setRadius(argi("radius"));
stabilizer->setInpainter(inpainters_);
}
if (arg("output") != "no")
outputPath = arg("output");
quietMode = argb("quiet");
run();
}
catch (const exception &e)
{
cout << "error: " << e.what() << endl;
stabilizedFrames.release();
return -1;
}
stabilizedFrames.release();
return 0;
}
MotionModel motionModel(const string &str)
{
if (str == "transl")
return MM_TRANSLATION;
if (str == "transl_and_scale")
return MM_TRANSLATION_AND_SCALE;
if (str == "rigid")
return MM_RIGID;
if (str == "similarity")
return MM_SIMILARITY;
if (str == "affine")
return MM_AFFINE;
if (str == "homography")
return MM_HOMOGRAPHY;
throw runtime_error("unknown motion model: " + str);
}