项目意义：当你看几十分钟的足球转播比赛视频，尤其是这个视频刚开始还有一段与比赛无关的片段，然后视频中比赛还有跳跃的现象，就是比如说，比赛进行到13分50秒的时候，下一帧突然变成20分，你很苦恼，你不知道视频时间如何和比赛时间进行对应，尤其是你想知道比分变动的时候，此刻比赛为第几秒。本项目讲解我进行记分牌信息识别的工作的大致原理。在下个项目中我会完整的用argparse封装好，并且完整的实现，尽情期待

1. 视频抽帧

以5帧每秒对于某个比赛视频进行抽帧，实现函数为extract_frames

import os
def extract_frames(video_name, out_folder, fps=5):
    if os.path.exists(out_folder):
        os.system('rm -rf ' + out_folder + '/*')
        os.system('rm -rf ' + out_folder)
    os.makedirs(out_folder)
    cmd = 'ffmpeg -v 0 -i %s -r %d -q 0 %s/%s.jpg' % (video_name, fps,
                                                      out_folder, '%08d')
    os.system(cmd)

#1. 先创建文件夹soccer_image
import os
if not os.path.isdir("soccer_image1"):
    os.mkdir("soccer_image1")

    #2.  150323 西甲  巴薩VS皇馬 1st.mp4重命名成soccer1.mp4
    os.rename("data/data161295/150323 西甲  巴薩VS皇馬 1st.mp4","data/data161295/soccer1.mp4")
    extract_frames("data/data161295/soccer1.mp4","soccer_image1")

#1. 先创建文件夹soccer_image
import os
if not os.path.isdir("soccer_image2"):
    os.mkdir("soccer_image2")

    extract_frames("data/data161295/1_720p.mkv","soccer_image2")

import os
#soccer1.mp4一共51分42秒,(51*60+42)*5=15510左右数量的图片
dirs = sorted(os.listdir("soccer_image1"))

print("soccer_img1文件夹中图片数量",len(dirs),dirs[:10])

soccer_img1文件夹中图片数量 15515 ['00000001.jpg', '00000002.jpg', '00000003.jpg', '00000004.jpg', '00000005.jpg', '00000006.jpg', '00000007.jpg', '00000008.jpg', '00000009.jpg', '00000010.jpg']

2. 记分牌Mask提取

意义：PaddleOCR本身自带文字检测，但是由于场上摄像机拍到很多杂元素，广告牌等，所以有了记分牌Mask可以很大程度减少干扰。

2.1 输入

2.2 Mask效果

2.3 算法思路

由于足球场有统一的主体颜色，我于是就可以首先提取图片的几种主体颜色，然后逐个像素判断是否属于这几种主体颜色中，这样就可以把大部分的足球场给去掉了，因为足球场大概率为主体颜色，但是记分牌可以被保留。但是此刻还有观众席，球员等元素。但是随着摄像机的移动，球员的跑动，将几个mask相乘，就可以保留记分牌MASK。至于电视台标在后续的处理中会去掉的

2.4 代码部分

from PIL import Image
import numpy as np
import cv2

def func_mask(infile):
    def get_dominant_colors(infile):
        image = Image.open(infile)
        # 缩小图片，否则计算机压力太大
        small_image = image.resize((80, 80))
        result = small_image.convert(
            "P", palette=Image.ADAPTIVE, colors=10
        )
        # # 找到主要的颜色
        palette = result.getpalette()
        # print(len(palette))
        color_counts = sorted(result.getcolors(), reverse=True)
        colors = list()

        for i in range(4):
            palette_index = color_counts[i][1]
            dominant_color = palette[palette_index * 3 : palette_index * 3 + 3]
            colors.append(tuple(dominant_color))

        return colors

    img = cv2.cvtColor(cv2.imread(infile, cv2.IMREAD_COLOR),cv2.COLOR_BGR2RGB)
    # img = cv2.imread(image_path, cv2.COLOR_BGR2HSV)
    # print(img.shape)
    main_color = get_dominant_colors(infile)
    # print(main_color)
    mask0 = np.zeros_like(img[:,:,0])
    for i in range(len(main_color)):
        color_Low = np.array(main_color[i])-20
        color_high = np.array(main_color[i])+20
        # print("color_low",color_Low)
        # print("color_high",color_high)
        mask = cv2.inRange(img,color_Low,color_high) # 该像素在范围内为255，不在则为0
        mask0 += mask
        # break
    img_mask =1 - mask0/255
    return img_mask

img_mask0 = func_mask("test/00003396.jpg")
img_mask1 = func_mask("test/00003305.jpg")
img_mask2 = func_mask("test/00003308.jpg")
img_mask3 = func_mask("test/00003331.jpg")

img_mask = img_mask0 * img_mask1 * img_mask2*img_mask3
img_mask = np.stack([img_mask,img_mask,img_mask],axis =-1)

# img_mask =1 - mask0/255
image_path = r"test/00003396.jpg"
img = cv2.cvtColor(cv2.imread(image_path, cv2.IMREAD_COLOR),cv2.COLOR_BGR2RGB)
output = img_mask*img
cv2.imwrite("mask.jpg",img_mask*255)

# print(img.shape)
output =output.astype(np.float32)
# # output = 
output =  cv2.cvtColor(output,cv2.COLOR_BGR2RGB)
cv2.imwrite("mask_img.jpg",output)

True

3. Paddleocr进行文字识别，进一步从记分牌提取信息

3.1 初步使用PaddleOCR的一个结果展示

3.2 代码测试

1. 先使用命令行
2. 再使用代码使用OCR

!pip install "paddleocr>=2.0.1" # 推荐使用2.0.1+版本

Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple
Collecting paddleocr>=2.0.1
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/f2/13/93d5fa0c96111322065b6ad2cb8818c674b42dc0f2411ad87a437021665c/paddleocr-2.5.0.3-py3-none-any.whl (334 kB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m334.7/334.7 kB[0m [31m18.0 MB/s[0m eta [36m0:00:00[0m
[?25hCollecting opencv-contrib-python==4.4.0.46
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/08/51/1e0a206dd5c70fea91084e6f43979dc13e8eb175760cc7a105083ec3eb68/opencv_contrib_python-4.4.0.46-cp37-cp37m-manylinux2014_x86_64.whl (55.7 MB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m55.7/55.7 MB[0m [31m8.6 MB/s[0m eta [36m0:00:00[0m:00:01[0m00:01[0m
[?25hRequirement already satisfied: openpyxl in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from paddleocr>=2.0.1) (3.0.5)
Collecting lxml
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/12/60/221cde1b87fcbc9260f92f939d99891ec151cf2ee3849ba909f02b614686/lxml-4.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (6.4 MB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m6.4/6.4 MB[0m [31m17.2 MB/s[0m eta [36m0:00:00[0m00:01[0m:00:01[0m
[?25hCollecting shapely
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/d1/ec/3038263d69a0065d3ab6944ae839f5f00896efd29b13ae62d73c00345b95/Shapely-1.8.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.0 MB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m2.0/2.0 MB[0m [31m68.4 MB/s[0m eta [36m0:00:00[0m
[?25hRequirement already satisfied: numpy in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from paddleocr>=2.0.1) (1.19.5)
Requirement already satisfied: cython in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from paddleocr>=2.0.1) (0.29)
Collecting premailer
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/b1/07/4e8d94f94c7d41ca5ddf8a9695ad87b888104e2fd41a35546c1dc9ca74ac/premailer-3.10.0-py2.py3-none-any.whl (19 kB)
Collecting attrdict
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/ef/97/28fe7e68bc7adfce67d4339756e85e9fcf3c6fd7f0c0781695352b70472c/attrdict-2.0.1-py2.py3-none-any.whl (9.9 kB)
Collecting lmdb
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/4d/cf/3230b1c9b0bec406abb85a9332ba5805bdd03a1d24025c6bbcfb8ed71539/lmdb-1.3.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (298 kB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m298.8/298.8 kB[0m [31m13.4 MB/s[0m eta [36m0:00:00[0m
[?25hRequirement already satisfied: tqdm in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from paddleocr>=2.0.1) (4.27.0)
Collecting python-Levenshtein
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/2a/dc/97f2b63ef0fa1fd78dcb7195aca577804f6b2b51e712516cc0e902a9a201/python-Levenshtein-0.12.2.tar.gz (50 kB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m50.5/50.5 kB[0m [31m1.8 MB/s[0m eta [36m0:00:00[0m
[?25h  Preparing metadata (setup.py) ... [?25ldone
[?25hCollecting imgaug==0.4.0
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/66/b1/af3142c4a85cba6da9f4ebb5ff4e21e2616309552caca5e8acefe9840622/imgaug-0.4.0-py2.py3-none-any.whl (948 kB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m948.0/948.0 kB[0m [31m5.9 MB/s[0m eta [36m0:00:00[0ma [36m0:00:01[0m
[?25hRequirement already satisfied: visualdl in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from paddleocr>=2.0.1) (2.3.0)
Collecting pyclipper
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/5b/88/737eb2291aad95d77765b8bc6b4cb2a59b69e72b5d1922c7bab9a0979cdb/pyclipper-1.3.0.post3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (604 kB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m604.2/604.2 kB[0m [31m22.8 MB/s[0m eta [36m0:00:00[0m
[?25hCollecting scikit-image
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/2d/ba/63ce953b7d593bd493e80be158f2d9f82936582380aee0998315510633aa/scikit_image-0.19.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (13.5 MB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m13.5/13.5 MB[0m [31m19.4 MB/s[0m eta [36m0:00:00[0m00:01[0m00:01[0m
[?25hRequirement already satisfied: scipy in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from imgaug==0.4.0->paddleocr>=2.0.1) (1.6.3)
Requirement already satisfied: Pillow in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from imgaug==0.4.0->paddleocr>=2.0.1) (8.2.0)
Requirement already satisfied: matplotlib in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from imgaug==0.4.0->paddleocr>=2.0.1) (2.2.3)
Requirement already satisfied: six in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from imgaug==0.4.0->paddleocr>=2.0.1) (1.16.0)
Requirement already satisfied: opencv-python in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from imgaug==0.4.0->paddleocr>=2.0.1) (4.1.1.26)
Requirement already satisfied: imageio in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from imgaug==0.4.0->paddleocr>=2.0.1) (2.6.1)
Requirement already satisfied: packaging>=20.0 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from scikit-image->paddleocr>=2.0.1) (21.3)
Requirement already satisfied: networkx>=2.2 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from scikit-image->paddleocr>=2.0.1) (2.4)
Collecting tifffile>=2019.7.26
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/d8/38/85ae5ed77598ca90558c17a2f79ddaba33173b31cf8d8f545d34d9134f0d/tifffile-2021.11.2-py3-none-any.whl (178 kB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m178.9/178.9 kB[0m [31m9.5 MB/s[0m eta [36m0:00:00[0m
[?25hCollecting PyWavelets>=1.1.1
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/ae/56/4441877073d8a5266dbf7b04c7f3dc66f1149c8efb9323e0ef987a9bb1ce/PyWavelets-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.4 MB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m6.4/6.4 MB[0m [31m18.6 MB/s[0m eta [36m0:00:00[0m00:01[0m:00:01[0m
[?25hRequirement already satisfied: et-xmlfile in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from openpyxl->paddleocr>=2.0.1) (1.0.1)
Requirement already satisfied: jdcal in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from openpyxl->paddleocr>=2.0.1) (1.4.1)
Requirement already satisfied: cachetools in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from premailer->paddleocr>=2.0.1) (4.0.0)
Collecting cssutils
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/49/49/1f2eb8f701ae0c9be4de912fdb48197da32fb8a8684693a878cbea25a813/cssutils-2.5.1-py3-none-any.whl (399 kB)
[2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m399.7/399.7 kB[0m [31m57.5 MB/s[0m eta [36m0:00:00[0m
[?25hCollecting cssselect
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/3b/d4/3b5c17f00cce85b9a1e6f91096e1cc8e8ede2e1be8e96b87ce1ed09e92c5/cssselect-1.1.0-py2.py3-none-any.whl (16 kB)
Requirement already satisfied: requests in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from premailer->paddleocr>=2.0.1) (2.24.0)
Requirement already satisfied: setuptools in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from python-Levenshtein->paddleocr>=2.0.1) (56.2.0)
Requirement already satisfied: bce-python-sdk in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from visualdl->paddleocr>=2.0.1) (0.8.53)
Requirement already satisfied: flask>=1.1.1 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from visualdl->paddleocr>=2.0.1) (1.1.1)
Requirement already satisfied: Flask-Babel>=1.0.0 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from visualdl->paddleocr>=2.0.1) (1.0.0)
Requirement already satisfied: pandas in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from visualdl->paddleocr>=2.0.1) (1.1.5)
Requirement already satisfied: protobuf>=3.11.0 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from visualdl->paddleocr>=2.0.1) (3.20.0)
Requirement already satisfied: Jinja2>=2.10.1 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from flask>=1.1.1->visualdl->paddleocr>=2.0.1) (3.0.0)
Requirement already satisfied: Werkzeug>=0.15 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from flask>=1.1.1->visualdl->paddleocr>=2.0.1) (0.16.0)
Requirement already satisfied: itsdangerous>=0.24 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from flask>=1.1.1->visualdl->paddleocr>=2.0.1) (1.1.0)
Requirement already satisfied: click>=5.1 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from flask>=1.1.1->visualdl->paddleocr>=2.0.1) (7.0)
Requirement already satisfied: pytz in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from Flask-Babel>=1.0.0->visualdl->paddleocr>=2.0.1) (2019.3)
Requirement already satisfied: Babel>=2.3 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from Flask-Babel>=1.0.0->visualdl->paddleocr>=2.0.1) (2.8.0)
Requirement already satisfied: decorator>=4.3.0 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from networkx>=2.2->scikit-image->paddleocr>=2.0.1) (4.4.2)
Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from packaging>=20.0->scikit-image->paddleocr>=2.0.1) (3.0.9)
Requirement already satisfied: pycryptodome>=3.8.0 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from bce-python-sdk->visualdl->paddleocr>=2.0.1) (3.9.9)
Requirement already satisfied: future>=0.6.0 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from bce-python-sdk->visualdl->paddleocr>=2.0.1) (0.18.0)
Requirement already satisfied: importlib-metadata in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from cssutils->premailer->paddleocr>=2.0.1) (4.2.0)
Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from matplotlib->imgaug==0.4.0->paddleocr>=2.0.1) (1.1.0)
Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from matplotlib->imgaug==0.4.0->paddleocr>=2.0.1) (2.8.2)
Requirement already satisfied: cycler>=0.10 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from matplotlib->imgaug==0.4.0->paddleocr>=2.0.1) (0.10.0)
Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from requests->premailer->paddleocr>=2.0.1) (3.0.4)
Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from requests->premailer->paddleocr>=2.0.1) (2019.9.11)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from requests->premailer->paddleocr>=2.0.1) (1.25.6)
Requirement already satisfied: idna<3,>=2.5 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from requests->premailer->paddleocr>=2.0.1) (2.8)
Requirement already satisfied: MarkupSafe>=2.0.0rc2 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from Jinja2>=2.10.1->flask>=1.1.1->visualdl->paddleocr>=2.0.1) (2.0.1)
Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from importlib-metadata->cssutils->premailer->paddleocr>=2.0.1) (4.3.0)
Requirement already satisfied: zipp>=0.5 in /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages (from importlib-metadata->cssutils->premailer->paddleocr>=2.0.1) (3.8.1)
Building wheels for collected packages: python-Levenshtein
  Building wheel for python-Levenshtein (setup.py) ... [?25ldone
[?25h  Created wheel for python-Levenshtein: filename=python_Levenshtein-0.12.2-cp37-cp37m-linux_x86_64.whl size=171679 sha256=4a9f1491f08a652a862ad10a6437adb44b0f334df6bc469e8dfffe7f12bbe109
  Stored in directory: /home/aistudio/.cache/pip/wheels/38/b9/a4/3729726160fb103833de468adb5ce019b58543ae41d0b0e446
Successfully built python-Levenshtein
Installing collected packages: pyclipper, lmdb, tifffile, shapely, PyWavelets, python-Levenshtein, opencv-contrib-python, lxml, cssselect, attrdict, scikit-image, cssutils, premailer, imgaug, paddleocr
Successfully installed PyWavelets-1.3.0 attrdict-2.0.1 cssselect-1.1.0 cssutils-2.5.1 imgaug-0.4.0 lmdb-1.3.0 lxml-4.9.1 opencv-contrib-python-4.4.0.46 paddleocr-2.5.0.3 premailer-3.10.0 pyclipper-1.3.0.post3 python-Levenshtein-0.12.2 scikit-image-0.19.3 shapely-1.8.2 tifffile-2021.11.2

[1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip available: [0m[31;49m22.1.2[0m[39;49m -> [0m[32;49m22.2.1[0m
[1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpip install --upgrade pip[0m

#命令行测试paddleocr 效果
# !paddleocr --image_dir mask_img.jpg --use_angle_cls true --use_gpu false
!paddleocr --help

from paddleocr import PaddleOCR, draw_ocr

# Paddleocr目前支持的多语言语种可以通过修改lang参数进行切换
# 例如`ch`, `en`, `fr`, `german`, `korean`, `japan`
'''
use_angle_cls = False 表示我不需要启用旋转，因为本来记分牌就是横的
'''
ocr = PaddleOCR(use_angle_cls=False, lang="ch")  # need to run only once to download and load model into memory

img_path = 'mask_img.jpg'

result = ocr.ocr(img_path, cls=True)
for line in result:
    if line[1][1]>0.8: #自己手工设置一个置信度阈值
        print(line)
len(line)

# # 显示结果
# from PIL import Image

# image = Image.open(img_path).convert('RGB')
# boxes = [line[0] for line in result]
# txts = [line[1][0] for line in result]
# scores = [line[1][1] for line in result]
# im_show = draw_ocr(image, boxes, txts, scores, font_path='STKAITI.TTF')#一定要指定font_path，不然会报错
# im_show = Image.fromarray(im_show)
# im_show.save('result.jpg')

4. 记分牌提取加Paddleocr大致整合形成基础版

4.1 效果展示

注意点：

1. 需要指定一个位置把电视台的的台标给遮盖掉

2. 我把paddleocr解压至PaddleOCR2_5,这样在本地我比较安心一点

3. PaddleOCR2_5/paddleocr.py

[‘直播西甲联赛第28轮’, ‘04:46’, ‘巴塞罗那0：0皇马’]

4.2 代码展示

!unzip -qo PaddleOCR2_5.zip

# import os

if not os.path.isdir("PaddleOCR2_5"):
    os.rename("PaddleOCR-release-2.5","PaddleOCR2_5")

from PIL import Image
import numpy as np
import cv2

def func_mask(infile):
    def get_dominant_colors(infile):
        image = Image.open(infile)
        # 缩小图片，否则计算机压力太大
        small_image = image.resize((80, 80))
        result = small_image.convert(
            "P", palette=Image.ADAPTIVE, colors=10
        )
        # # 找到主要的颜色
        palette = result.getpalette()
        # print(len(palette))
        color_counts = sorted(result.getcolors(), reverse=True)
        colors = list()
        color_num = 4 if 4<len(color_counts) else len(color_counts)
        for i in range(color_num):
            palette_index = color_counts[i][1]
            dominant_color = palette[palette_index * 3 : palette_index * 3 + 3]
            colors.append(tuple(dominant_color))
        return colors


    img = cv2.cvtColor(cv2.imread(infile, cv2.IMREAD_COLOR),cv2.COLOR_BGR2RGB)
    # img = cv2.imread(image_path, cv2.COLOR_BGR2HSV)
    # print(img.shape)
    main_color = get_dominant_colors(infile)
    # print(main_color)
    mask0 = np.zeros_like(img[:,:,0])
    for i in range(len(main_color)):
        color_Low = np.array(main_color[i])-20
        color_high = np.array(main_color[i])+20
        # print("color_low",color_Low)
        # print("color_high",color_high)
        mask = cv2.inRange(img,color_Low,color_high) # 该像素在范围内为255，不在则为0
        mask0 += mask
        # break
    img_mask =1 - mask0/255
    img_mask = img_mask.astype("int8").astype("float32")
    return img_mask

from PaddleOCR2_5 import PaddleOCR, draw_ocr
from PIL import Image


# Paddleocr目前支持的多语言语种可以通过修改lang参数进行切换
# 例如`ch`, `en`, `fr`, `german`, `korean`, `japan`
'''
use_angle_cls = False 表示我不需要启用旋转，因为本来记分牌就是横的
'''

##选取几张图片得到mask，进行相乘，进一步确认记分牌位置
img_mask0 = func_mask("test/00003396.jpg")
img_mask1 = func_mask("test/00003305.jpg")
img_mask2 = func_mask("test/00003308.jpg")
img_mask3 = func_mask("test/00003331.jpg")

img_mask = img_mask0 * img_mask1 * img_mask2*img_mask3
img_mask = np.stack([img_mask,img_mask,img_mask],axis =-1) #得到mask


### image_path为要进行记分牌检测的图片
image_path = r"test/00003331.jpg" 
# img = cv2.cvtColor(cv2.imread(image_path, cv2.IMREAD_COLOR),cv2.COLOR_BGR2RGB)
img = cv2.imread(image_path, cv2.IMREAD_COLOR)
output = img
# output = img_mask*img
output =output.astype(np.float32)
ocr = PaddleOCR(use_angle_cls=False, lang="ch")  # need to run only once to download and load model into memory

h,w = output.shape[:-1]
## 按比例制作去台标mask
qutaibiao_mask = np.ones([h,w,1])
qutaibiao_mask[:int(0.3*h),int(0.6*w):] = 0 #人工设置一个区域把台标区域给去掉
output = qutaibiao_mask*output
result = ocr.ocr(output, cls=True)

jifenpai_list = []
for line in result:
    if line[1][1]>0.8: #自己手工设置一个置信度阈值
        print(line)
        jifenpai_list.append(line[1][0])
print(jifenpai_list) #['直播西甲联赛第28轮', '04:46', '巴塞罗那0：0皇马']


# 显示结果
image = Image.open(image_path).convert('RGB')
boxes = [line[0] for line in result]
txts = [line[1][0] for line in result]
scores = [line[1][1] for line in result]
im_show = draw_ocr(image, boxes, txts, scores, font_path='STKAITI.TTF')#一定要指定font_path，不然会报错
im_show = Image.fromarray(im_show)
im_show.save('result1.jpg')

5. 进一步考虑

我们要实现的目标是让视频的时间可以和比赛时间对应起来，另外我们需要尽量处理成字典key-value的形式。

5.1 两个目标的详细讲解

5.1.1 目标1：视频时间和比赛时间对应

因为视频前面比如会有人物出场，会有一段非足球比赛的视频片段，然后中间的比赛进行时间与MP4视频时间对应（比如视频的第1分20秒为比赛开始20秒），因此我们必须要明白视频中什么时候开始出现记分牌。

那么我们很重要的一个步骤就是需要识别在这个视频中比赛是第几秒开始的:首先我们进行一个粗粒度的抽帧，然后进行一些操作，得到一个比赛具体开始的范围

5.1.2 目标2： 通过记分牌得到队名，比赛名，此刻比赛时间，双方比分

1. 队名我们通过在一个预设的列表中查找
2. 比赛名我们也会通过一个预设的比赛名列表中查找
3. 如果含有冒号和数字，则该位置为比赛时间或者双方比分，前后几秒中数字不变则为比分位置，反正隔一秒变一次就为比赛时间位置

5.2 算法流程：

5.2.1 得到视频第一次出现记分牌的时间

针对于比赛视频已经抽帧好的图片文件夹（一秒5帧），记分牌有时间元素（已经处理成1秒5帧的有序的一组图片），那么我先粗粒度抽取每隔10s（就是每顺序隔50张）抽一帧，当paddleocr识别到第a帧有时间（通过ocr识别图片上字，如果存在":"或“:”就继续用正则化判断是否存在时间要素），如果判断到第a张图片有时间，并且第（a+50）张也有时间(这个条件也可以去掉，因为有些比赛刚开始比赛的时间会隔一会儿被遮挡掉，从而导致问题)，但是第（a-50）张没有时间，就判断视频开始在第a-50帧到a帧中间开始。然后在在这50帧中间匹配时间就可精准定位。

5.2.2 得到基于主颜色的记分牌mask（可能包括电视台的标识，还有其他水印）

再在视频出现记分牌后选择每隔较远帧的图片得到mask（可能包括电视台的标识，还有其他水印）

5.2.3 得到队名，时间，比分

首先根据OCR文字检测加正则化得到的时间位置中心(x,y)，如果其他文字元素中心点位置距离时间中心点位置大于一个预设的数值（0.5*图片的宽），该元素即为非记分牌文字信息就可以忽略。其他记分牌文字元素信息就可判断队名（为预设的一个队名列表），比赛名（为一个预设的比赛列表），然后基于正则判断比分。

# import numpy as np
# a = np.arange(16).reshape([4,4])
# b = a.reshape([2,8]).reshape([4,4])
# print(a,b)

from PaddleOCR2_5 import PaddleOCR, draw_ocr
import os
import paddle
import numpy as np
import re
from PIL import Image
import cv2
import paddle.nn.functional as F
#soccer1.mp4一共51分42秒,(51*60+42)*5=15510左右数量的图片
folder_name = "soccer_image1"
dirs = sorted(os.listdir(folder_name))
# print(dirs)
seq_second = 10
fps = 5
dirs1 = dirs[::fps*seq_second] #每隔十秒选1张，毕竟1秒5帧
# print("dirs1",dirs1)
ocr = PaddleOCR(use_angle_cls=True, lang="ch")  # need to run only once to download and load model into memory

begin_video_time_fps = 0
# end_video_time_fps = 0
# return_end_match_time = False
temp = 0

#粗粒度判断记分牌时间出现的视频时间
for i in range(len(dirs1)):
    if dirs1[i].split(".")[-1] != "jpg":
        continue
    one_path = os.path.join(folder_name,dirs1[i])

    output = cv2.imread(one_path)

    result = ocr.ocr(output, cls=True)
    allwords_list = []
    # if result is not None:
    words_point_list = []
    for line in result:
        if line[1][1]>0.8: #自己手工设置一个置信度阈值
            # print(line)
            allwords_list.append(line[1][0])
    one_image_str = " ".join(allwords_list)
    # if "：" in one_image_str or ":" in one_image_str:
    #     pattern = r"\d\d[:,：]\d\d" 
    #     image_time = re.search(pattern, one_image_str, flags=0)
    #     if image_time is not None:
    #         temp +=1
    #         if temp == 2: #连续两次粗粒度都有时间元素
    #             begin_video_time_fps = i-1
    #             break
    # else:
    #     temp = 0
    if "：" in one_image_str or ":" in one_image_str:
        pattern = r"\d\d[:,：,.]\d\d" 
        image_time = re.search(pattern, one_image_str, flags=0)
        if image_time is not None:
            if begin_video_time_fps == 0:
                begin_video_time_fps = i
                break
# print(begin_video_time_fps)

#细粒度判断记分牌时间出现的视频时间
for j in range(begin_video_time_fps*fps*seq_second-fps*seq_second,begin_video_time_fps*fps*seq_second):
    one_path = os.path.join(folder_name,dirs[j])
    output = cv2.imread(one_path)
    result = ocr.ocr(output, cls=True)
    allwords_list = []
    # if result is not None:
    words_point_list = []
    for line in result:
        if line[1][1]>0.8: #自己手工设置一个置信度阈值
            # print(line)
            allwords_list.append(line[1][0])
    one_image_str = " ".join(allwords_list)
    if "：" in one_image_str or ":" in one_image_str:
        pattern = r"\d\d[:,：]\d\d" 
        image_time = re.search(pattern, one_image_str, flags=0)
        if image_time is not None:
            begin_video_time_fps = j
            print(j,one_image_str,"此刻为比赛时间"+image_time[0],sep='------')
            break
begin_video_time = begin_video_time_fps/fps #单位秒
begin_match_time = image_time[0]
print("这个比赛开始于该视频的",begin_video_time//60,"分",begin_video_time - begin_video_time//60*60,"秒","此刻比赛时间为",begin_match_time)
        

#自己写的文本匹配算法，需要保证足球队名按顺序提取就行
soccer_team = ["巴塞罗那","皇马"]
sentence = "巴塞罗那0：0皇马"
def text_match(word_list,sentence):
    return_words = []
    len_sen = len(sentence)
    begin_index = 0
    end_index = 0
    while True:
        temp = 0
        # print(sentence[begin_index:end_index])
        for one_word in word_list:
            if one_word in sentence[begin_index:end_index]:                
                return_words.append(one_word)
                begin_index = end_index
                temp +=1
                break
        if temp ==0:
            end_index+=1
        if end_index == len_sen+1:
            break
    return return_words
text_match(soccer_team,sentence)

['巴塞罗那', '皇马']

from PaddleOCR2_5 import PaddleOCR, draw_ocr
from PIL import Image
import math



##选取几张图片得到mask，进行相乘，进一步确认记分牌位置
folder_name = "soccer_image1"
img_mask = np.ones_like(cv2.imread(os.path.join(folder_name,dirs[6]))[:,:,0]).astype("float32")
h,w = img_mask.shape
print(begin_video_time_fps)
for k in range(4):
    index = begin_video_time_fps +40*k+5
    img_mask0 = func_mask(os.path.join(folder_name,dirs[index]))
    img_mask *= img_mask0

img_mask = np.stack([img_mask,img_mask,img_mask],axis =-1) #得到mask


### image_path为要进行记分牌检测的图片
image_path = r"soccer_image1/00010329.jpg" 
img = cv2.imread(image_path, cv2.IMREAD_COLOR)
# output = img
output = img_mask*img
output =output.astype(np.float32)
ocr = PaddleOCR(use_angle_cls=False, lang="ch")  # need to run only once to download and load model into memory


result = ocr.ocr(output, cls=True)
print(result)
time_pattern = r"\d\d[:,：,.]\d\d" 
point_pattern = r"\d[ ,:,：,-]+\d" 

for line in result:
    image_time = re.search(time_pattern,line[1][0], flags=0)
    if  image_time is not None: 
        time_location = line[0]
        x = sum([one_local[0] for one_local in time_location])/4
        y = sum([one_local[1] for one_local in time_location])/4
        time_center_local = (x,y)
        break
jifenpai_list = []
jifenpai_dict = {}
jifenpai_dict["足球队伍"] = []
soccer_team = ["巴塞罗那","皇马","CHE","BUR" ]

for line in result:
    key_local = line[0]
    x = sum([one_local[0] for one_local in key_local])/4
    y = sum([one_local[1] for one_local in key_local])/4
    key_center_local = (x,y)
    distance_with_time = math.sqrt(sum([math.pow((time_center_local[i]-key_center_local[i]),2) for i in range(2)]))
    if line[1][1]>0.8 and distance_with_time<0.5*w: #自己手工设置一个置信度阈值
        print(line)
        jifenpai_list.append(line[1][0])
        if text_match(soccer_team,line[1][0]):
            jifenpai_dict["足球队伍"] += text_match(soccer_team,line[1][0])
        if re.search(time_pattern,line[1][0], flags=0) is not None:
            jifenpai_dict["时间"] = re.search(time_pattern,line[1][0], flags=0)[0]
        if  re.search(time_pattern,line[1][0], flags=0) is None and  re.search(point_pattern,line[1][0], flags=0) is not None:
            jifenpai_dict["比分"] =  re.search(point_pattern,line[1][0], flags=0)[0]
        
print(jifenpai_dict) #['直播西甲联赛第28轮', '04:46', '巴塞罗那0：0皇马']


# 显示结果
image = Image.open(image_path).convert('RGB')
boxes = [line[0] for line in result]
txts = [line[1][0] for line in result]
scores = [line[1][1] for line in result]
im_show = draw_ocr(image, boxes, txts, scores, font_path='STKAITI.TTF')#一定要指定font_path，不然会报错
im_show = Image.fromarray(im_show)
im_show.save('result1.jpg')

6. 遇到问题描述

1. 比分Paddleocr默认模型识别不出来,换别的OCR模型
2. 记分牌时间中间冒号被识别成点，我在正则表达式中改变了