[fix-data][s] Fixing up the data issue format - #34

datasets · Nov 7, 2024 · 46fb1e1 · 46fb1e1
1 parent 9db194e
commit 46fb1e1
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/data/code-list.csv b/data/code-list.csv
@@ -61393,7 +61393,6 @@ x,ZA,NGQ,Port of Ngqura,Port of Ngqura,EC,XX,1----6--,2307,,3346S 02540E,
 ,ZW,VFA,Victoria Falls,Victoria Falls,,AI,---4----,9501,,,
 ,ZW,ZMZ,Zimbabwe,Zimbabwe,MV,RL,1-3-----,1401,,2016S 03055E,
 ,ZW,ZVS,Zvishavane,Zvishavane,MI,RL,--3-----,1207,,2020S 03002E,
-Change,Country,Location,Name,NameWoDiacritics,Subdivision,Function,Status,Date,IATA,Coordinates,Remarks
 ,AO,FBY,Farta Bay,Farta Bay,,1-------,RQ,0901,,1237S 01312E,
 ,AO,FLT,Futila Terminal,Futila Terminal,,1-------,RQ,0901,,0527S 01211E,
 ,AO,GIM,Gimboa,Gimboa,,1-------,RL,1107,,0732S 01210E,

diff --git a/scripts/prepare.py b/scripts/prepare.py
@@ -91,7 +91,10 @@ def process(extracted_files):
     codelist_df  = pd.DataFrame(codelist_list) #)=pd.concat(codelist_list)
     codelist_df = codelist_df.reindex(columns=['Change', 'Country', 'Location', 'Name', 'NameWoDiacritics', 'Subdivision',
                         'Status', 'Function', 'Date', 'IATA', 'Coordinates', 'Remarks'])
+    # Keep only rows where 'Country' values are empty, 1 character, or exactly 2 characters
+    codelist_df = codelist_df[codelist_df['Country'].str.len().fillna(0).between(0, 2)]
     codelist_df.to_csv(f"data/code-list.csv", index=False)
+
     alias_df.to_csv(f"data/alias.csv", index=False)
     print("Processed and saved UNLOCODE files")
     return