diff --git a/.github/diff_pot_files.py b/.github/diff_pot_files.py
new file mode 100644
index 0000000..53082f9
--- /dev/null
+++ b/.github/diff_pot_files.py
@@ -0,0 +1,29 @@
+#! /usr/bin/env python3
+
+"""
+Super small script for github action to detect if 2 .pot files have changed for
+github/workflows/auto_messages_pot.yml
+"""
+
+import sys
+from babel.messages.pofile import PoFileParser
+
+
+def load_pot_file(file_path):
+    poparser = PoFileParser({})
+    poparser.parse(open(file_path))
+    return poparser.catalog
+
+
+def main():
+    file_1 = load_pot_file(sys.argv[1])
+    file_2 = load_pot_file(sys.argv[2])
+
+    if [x for x in file_1.keys() if x] == [x for x in file_2.keys() if x]:
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/auto_messages_pot.yml b/.github/workflows/auto_messages_pot.yml
index 5a00f73..c290b6f 100644
--- a/.github/workflows/auto_messages_pot.yml
+++ b/.github/workflows/auto_messages_pot.yml
@@ -18,6 +18,10 @@ jobs:
         run: |
           cd tools/readme_generator/
           pip install -r requirements.txt
+      - name: Save old messges.pot for diffing later
+        run: |
+          cd tools/readme_generator/
+          cp messages.pot messages.pot.old
       - name: Try to generate messages.pot
         run: |
           cd tools/readme_generator/
@@ -25,9 +29,7 @@ jobs:
       - shell: pwsh
         id: check_files_changed
         run: |
-          # Diff HEAD with the previous commit
-          $diff = git diff
-          $HasDiff = $diff.Length -gt 0
+          $HasDiff = python .github/diff_pot_files.py tools/readme_generator/messages.pot.old tools/readme_generator/messages.pot
           Write-Host "::set-output name=files_changed::$HasDiff"
       - name: Create Pull Request
         if: steps.check_files_changed.outputs.files_changed == 'true'