avformat/rpl: Fix check for negative values

Fixes: signed integer overflow: 10 * -1923267925333400000 cannot be represented in type 'int64_t' (aka 'long')
Fixes: 378891963/clusterfuzz-testcase-minimized-fuzzer_loadfile_direct-5714338935013376
Found-by: ossfuzz
Reported-by: Kacper Michajlow <[email protected]>
Signed-off-by: Michael Niedermayer <[email protected]>
(cherry picked from commit eab65379bf89c55d8ec4bc6f00e04f15b37d3d85)
Signed-off-by: Michael Niedermayer <[email protected]>

diff --git a/libavformat/rpl.c b/libavformat/rpl.c
index ac82940b7a38fde9ffb0df13162cabef0406b759..5f1eaa2957917296bf1be6768811b123a098ba5a 100644 (file)
--- a/libavformat/rpl.c
+++ b/libavformat/rpl.c
@@ -101,7 +101,7 @@ static AVRational read_fps(const char* line, int* error)
         line++;
     for (; *line>='0' && *line<='9'; line++) {
         // Truncate any numerator too large to fit into an int64_t
-        if (num > (INT64_MAX - 9) / 10 || den > INT64_MAX / 10)
+        if (num > (INT64_MAX - 9) / 10ULL || den > INT64_MAX / 10ULL)
             break;
         num  = 10 * num + (*line - '0');
         den *= 10;