Uniq in subdivision table

datasets · Feb 16, 2024 · 750e861 · 750e861
1 parent b65f2e1
commit 750e861
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 5 deletions.
diff --git a/data/subdivision-codes.csv b/data/subdivision-codes.csv
@@ -1756,7 +1756,6 @@ IN,HP,Himāchal Pradesh
 IN,HR,Haryāna
 IN,JH,Jhārkhand
 IN,JK,Jammu and Kashmīr
-IN,JK,Jammu and Kashmīr
 IN,KA,Karnātaka
 IN,KL,Kerala
 IN,LA,Ladākh
@@ -2450,7 +2449,6 @@ MA,CAS,Casablanca [Dar el Beïda]*
 MA,CHE,Chefchaouene
 MA,CHI,Chichaoua
 MA,CHT,Chtouka-Ait Baha
-MA,CHT,Chtouka-Ait Baha
 MA,DRI, Driouch
 MA,ERR,Errachidia
 MA,ESI,Essaouira
@@ -2640,7 +2638,6 @@ MK,202,Vinica
 MK,203,Delčevo
 MK,204,Zrnovci
 MK,205,Karbinci
-MK,205,Karbinci
 MK,206,Kočani
 MK,207,Makedonska Kamenica
 MK,208,Pehčevo

diff --git a/datapackage.json b/datapackage.json
@@ -1,7 +1,7 @@
 {
   "name": "un-locode",
   "title": "UN-LOCODE Codelist",
-  "version": "2023.2.0",
+  "version": "2023.2.1",
   "licenses": [
     {
       "name": "ODC-PDDL-1.0",

diff --git a/scripts/prepare_edition.sh b/scripts/prepare_edition.sh
@@ -12,6 +12,7 @@ done
 gawk -v RS='"[^"]*"' -v ORS= '{gsub(/\n/, " ", RT); print $0  RT}' mdb_SubdivisionCodes_out.csv > subdivision-codes.tmp.csv
 sed -i 's/ \{2,\}/ /g;s/[[:blank:]]*$//' subdivision-codes.tmp.csv
 sed -i 's/\t/ /g' subdivision-codes.tmp.csv
+uniq subdivision-codes.tmp.csv > subdivision-codes.tmp2.csv
 
 gawk -v RS='"' 'NR % 2 == 0 { gsub(/\n/, "") } { printf("%s%s", $0, RT) }'  "mdb_${filename}_out.csv" > code-list.tmp.csv #remove newlines in data
 csvgrep -c 3 -r "^$" -i code-list.tmp.csv > code-list.tmp2.csv #remove country headers
@@ -22,5 +23,5 @@ mv code-list.csv data/code-list.csv
 mv mdb_CountryCodes_out.csv data/country-codes.csv
 mv mdb_FunctionClassifiers_out.csv data/function-classifiers.csv
 mv mdb_StatusIndicators_out.csv data/status-indicators.csv
-mv subdivision-codes.tmp.csv data/subdivision-codes.csv
+mv subdivision-codes.tmp2.csv data/subdivision-codes.csv
 rm *.csv