-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathOLCI.html
997 lines (909 loc) · 77 KB
/
OLCI.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta content="width=device-width, initial-scale=1.0" name="viewport">
<title>OLCI</title>
<meta content="" name="description">
<meta content="" name="keywords">
<!-- Favicons -->
<link href="assets/img/Favicon-1.png" rel="icon">
<link href="assets/img/Favicon-1.png" rel="apple-touch-icon">
<!-- Google Fonts -->
<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,300i,400,400i,600,600i,700,700i|Raleway:300,300i,400,400i,500,500i,600,600i,700,700i|Poppins:300,300i,400,400i,500,500i,600,600i,700,700i" rel="stylesheet">
<!-- Vendor CSS Files -->
<link href="assets/vendor/aos/aos.css" rel="stylesheet">
<link href="assets/vendor/bootstrap/css/bootstrap.min.css" rel="stylesheet">
<link href="assets/vendor/bootstrap-icons/bootstrap-icons.css" rel="stylesheet">
<link href="assets/vendor/boxicons/css/boxicons.min.css" rel="stylesheet">
<link href="assets/vendor/glightbox/css/glightbox.min.css" rel="stylesheet">
<link href="assets/vendor/swiper/swiper-bundle.min.css" rel="stylesheet">
<!-- Creating a python code section-->
<link rel="stylesheet" href="assets/css/prism.css">
<script src="assets/js/prism.js"></script>
<!-- Template Main CSS File -->
<link href="assets/css/style.css" rel="stylesheet">
<!-- To set the icon, visit https://fontawesome.com/account-->
<script src="https://kit.fontawesome.com/5d25c1efd3.js" crossorigin="anonymous"></script>
<!-- end of icon-->
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js?config=TeX-MML-AM_CHTML">
</script>
<!-- Include the highlight.js library -->
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.3.1/styles/default.min.css">
<!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.3.1/highlight.min.js"></script> -->
<!-- =======================================================
* Template Name: iPortfolio
* Updated: Sep 18 2023 with Bootstrap v5.3.2
* Template URL: https://bootstrapmade.com/iportfolio-bootstrap-portfolio-websites-template/
* Author: BootstrapMade.com
* License: https://bootstrapmade.com/license/
======================================================== -->
</head>
<body>
<!-- ======= Mobile nav toggle button ======= -->
<i class="bi bi-list mobile-nav-toggle d-xl-none"></i>
<!-- ======= Header ======= -->
<header id="header">
<div class="d-flex flex-column">
<div class="profile">
<img src="assets/img/myphoto.jpeg" alt="" class="img-fluid rounded-circle">
<h1 class="text-light"><a href="index.html">Arun</a></h1>
<div class="social-links mt-3 text-center">
<a href="https://www.linkedin.com/in/arunp77/" target="_blank" class="linkedin"><i class="bx bxl-linkedin"></i></a>
<a href="https://github.com/arunp77" target="_blank" class="github"><i class="bx bxl-github"></i></a>
<a href="https://twitter.com/arunp77_" target="_blank" class="twitter"><i class="bx bxl-twitter"></i></a>
<a href="https://www.instagram.com/arunp77/" target="_blank" class="instagram"><i class="bx bxl-instagram"></i></a>
<a href="https://arunp77.medium.com/" target="_blank" class="medium"><i class="bx bxl-medium"></i></a>
</div>
</div>
<nav id="navbar" class="nav-menu navbar">
<ul>
<li><a href="index.html#hero" class="nav-link scrollto active"><i class="bx bx-home"></i> <span>Home</span></a></li>
<li><a href="index.html#about" class="nav-link scrollto"><i class="bx bx-user"></i> <span>About</span></a></li>
<li><a href="index.html#resume" class="nav-link scrollto"><i class="bx bx-file-blank"></i> <span>Resume</span></a></li>
<li><a href="index.html#portfolio" class="nav-link scrollto"><i class="bx bx-book-content"></i> <span>Portfolio</span></a></li>
<li><a href="index.html#skills-and-tools" class="nav-link scrollto"><i class="bx bx-wrench"></i> <span>Skills and Tools</span></a></li>
<li><a href="index.html#language" class="nav-link scrollto"><i class="bi bi-menu-up"></i> <span>Languages</span></a></li>
<li><a href="index.html#awards" class="nav-link scrollto"><i class="bi bi-award-fill"></i> <span>Awards</span></a></li>
<li><a href="index.html#professionalcourses" class="nav-link scrollto"><i class="bx bx-book-alt"></i> <span>Professional Certification</span></a></li>
<li><a href="index.html#publications" class="nav-link scrollto"><i class="bx bx-news"></i> <span>Publications</span></a></li>
<li><a href="index.html#extra-curricular" class="nav-link scrollto"><i class="bx bx-rocket"></i> <span>Extra-Curricular Activities</span></a></li>
<!-- <li><a href="#contact" class="nav-link scrollto"><i class="bx bx-envelope"></i> <span>Contact</span></a></li> -->
</ul>
</nav><!-- .nav-menu -->
</div>
</header><!-- End Header -->
<main id="main">
<!-- ======= Breadcrumbs ======= -->
<section id="breadcrumbs" class="breadcrumbs">
<div class="container">
<div class="d-flex justify-content-between align-items-center">
<h2>Remote sensing</h2>
<ol>
<li><a href="Remote-sensing-content.html" class="clickable-box">Content section</a></li>
<li><a href="index.html#portfolio" class="clickable-box">Portfolio section</a></li>
</ol>
</div>
</div>
</section><!-- End Breadcrumbs -->
<!------ right dropdown menu ------->
<div class="right-side-list">
<div class="dropdown">
<button class="dropbtn"><strong>Shortcuts:</strong></button>
<div class="dropdown-content">
<ul>
<li><a href="cloud-compute.html"><i class="fas fa-cloud"></i> Cloud</a></li>
<li><a href="AWS-GCP.html"><i class="fas fa-cloud"></i> AWS-GCP</a></li>
<li><a href="amazon-s3.html"><i class="fas fa-cloud"></i> AWS S3</a></li>
<li><a href="ec2-confi.html"><i class="fas fa-server"></i> EC2</a></li>
<li><a href="Docker-Container.html"><i class="fab fa-docker" style="color: rgb(29, 27, 27);"></i> Docker</a></li>
<li><a href="Jupyter-nifi.html"><i class="fab fa-python" style="color: rgb(34, 32, 32);"></i> Jupyter-nifi</a></li>
<li><a href="snowflake-task-stream.html"><i class="fas fa-snowflake"></i> Snowflake</a></li>
<li><a href="data-model.html"><i class="fas fa-database"></i> Data modeling</a></li>
<li><a href="sql-basics.html"><i class="fas fa-table"></i> SQL</a></li>
<li><a href="sql-basic-details.html"><i class="fas fa-database"></i> SQL</a></li>
<li><a href="Bigquerry-sql.html"><i class="fas fa-database"></i> Bigquery</a></li>
<li><a href="scd.html"><i class="fas fa-archive"></i> SCD</a></li>
<li><a href="sql-project.html"><i class="fas fa-database"></i> SQL project</a></li>
<!-- Add more subsections as needed -->
</ul>
</div>
</div>
</div>
<!-- ======= Portfolio Details Section ======= -->
<section id="portfolio-details" class="portfolio-details">
<div class="container">
<div class="row gy-4">
<h1>Oceanography : A remote sensing perspective</h1>
<div class="col-lg-8">
<div class="portfolio-details-slider swiper">
<div class="swiper-wrapper align-items-center">
<figure>
<img src="assets/img/remote-sensing/OLCI_S3A_chlora_LI_nolandmask_20201009.png" alt="" style="max-width: 80%; max-height: auto;">
<figcaption style="text-align: center;"><b>Reference:</b> <a href="https://eastcoast.coastwatch.noaa.gov/cw_olci_chl.php" target="_blank">OLCI Chlorophyll-a, NY-NJ Bight & Long Island Sound, Oct 9, 2020, 300 m</a> </figcaption>
</figure>
</div>
<div class="swiper-pagination"></div>
</div>
</div>
<div class="col-lg-4 grey-box">
<h3>Content</h3>
<ol>
<li><a href="#introduction">Introduction</a>
<ul>
<li><a href="#fundamental">Fundamentals of Oceanography</a></li>
<li><a href="#remote-key">Key Remote Sensing Instruments</a></li>
</ul>
</li>
<li><a href="#Sentinel-3">Sentinel 3 : Satellite</a>
<ul>
<li><a href="#instruments">Sentinel-3 Instruments</a></li>
<li><a href="#OCLI-module">OCLI-module</a>
<ul>
<li><a href="#olci-spectrum">OLCI spectrum</a></li>
<li><a href="#baseline">OLCI processing baseline</a></li>
<li><a href="#olci-product">OLCI Product</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#olci-sent-analysis">OLCI Sentinel-3 data (analysis)</a>
<ul>
<li><a href="#data-sentinel3">Accessing OLCI data via the EUMETSAT Data Store</a></li>
<li><a href="#OCLI-query">Querying OLCI file structure (Level-1B)</a></li>
</ul>
</li>
<li><a href="#reference">Reference</a></li>
</ol>
</div>
</div>
<!---------sections start here ------------>
<hr>
<section>
<div class="alert alert-success"> <i class="bi bi-arrow-right-circle"></i>
Currently developing this page. This is a part of a workshop organized by <a href="https://umd.instructure.com/courses/1356565" target="_blank">Operational Satellite Oceanography Workshop (2023)</a>. Few contents are part of what I learnt during the workshop.
</div>
<!----------------->
<h1 id="introduction">Introduction</h1>
Oceanography, the study of the physical, chemical, biological, and geological aspects of the ocean, plays a crucial role in understanding Earth's complex marine environment. This field leverages a variety of methodologies and technologies, including remote sensing, to collect, analyze, and interpret oceanic data.
<h3 id="fundamental">Fundamentals of Oceanography</h3>
<ol>
<li><b>Physical Oceanography: </b>Physical oceanography deals with the study of ocean currents, waves, and tides. It explores the interactions between the ocean and the atmosphere, which significantly influence climate patterns.</li>
<li><b>Chemical Oceanography: </b>Chemical oceanography focuses on the chemical composition of seawater and the biogeochemical cycles. It examines the sources and sinks of various chemical elements and compounds in the ocean.</li>
<li><b>Biological Oceanography: </b>Biological oceanography investigates the marine organisms and their interactions with the ocean environment. It covers the study of marine ecosystems, food webs, and the impact of human activities on marine life.</li>
<li><b>Geological Oceanography: </b>Geological oceanography examines the structure and composition of the ocean floor. It includes the study of plate tectonics, underwater volcanism, and sediment processes.</li>
</ol>
<h3 id="remote-key">Key Remote Sensing Instruments</h3>
<ul>
<li><b>Radiometers: </b> Measure the intensity of radiation, providing data on sea surface temperature and ocean color.</li>
<li><b>Radar Altimeters: </b> Measure sea surface height, which is crucial for understanding ocean circulation and sea level rise.</li>
<li><b>LIDAR (Light Detection and Ranging): </b> Uses laser pulses to measure the depth and topography of the ocean floor.</li>
</ul>
<p>Oceanography is an interdisciplinary field that utilizes advanced technologies, including remote sensing, to study and understand the ocean. By integrating data from various sources and using sophisticated data formats, oceanographers can monitor, analyze, and predict changes in the marine environment. This knowledge is crucial for managing marine resources, mitigating the impacts of climate change, and protecting ocean health.</p>
</section>
<section>
<h2 id="Sentinel-3">Sentinel 3 : Satellite</h2>
Sentinel-3 is an European Earth Observation satellite mission developed to support Copernicus ocean, land, atmospheric, emergency, security and cryospheric applications. It is jointly operated by ESA and EUMETSAT to deliver operational ocean and land observation services. There are two satellites lanuched: Sentinel-3A on 16 February 2016 Sentinel-3B on 25 April 2018 (a more detail on the satellite and various components are given at <a href="https://sentiwiki.copernicus.eu/web/sentinel-3" target="_blank">official page</a>).
<figure>
<img src="assets/img/remote-sensing/sentinel-3-instruments.png" alt="" style="max-width: 60%; max-height: auto;">
<figcaption style="text-align: center;"><b>Reference:</b> <a href="https://earth.esa.int/eogateway/documents/20142/1564943/The-Global-Monitoring-for-Environment-and-Security-GMES-Sentinel-3-mission.pdf" target="_blank">Sentinel-3 instruments</a> (page - 6 of the reference paper).</figcaption>
</figure>
<ul>
<li><b>Main objectives:</b> Systematically measures Earth's oceans, land, ice, and atmosphere.
<ul>
<li>Measure sea surface topography.</li>
<li>Measure sea and land surface temperature.</li>
<li>Measure ocean and land surface color.</li>
</ul>
</li>
<li><b>Additional applications: </b>Sea-level change & sea-surface temperature mapping, water quality management, sea-ice extent and thickness mapping and numerical ocean prediction; land-cover mapping, vegetation health monitoring; glacier monitoring; water resource monitoring; wildfire detection; numerical weather prediction.</li>
</li>
<li><b>Spectral bands: </b> (For more on theoretical basis on the Spectral Calibration details, see page number 13 in the following <a href="https://sentiwiki.copernicus.eu/__attachments/1672112/S3-RP-RAL-SL-102%20-%20SLSTR%20A%20FPA%20Spectral%20Calibration%20Function%202015%20-%201.0.pdf?inst-v=3ef77677-e18f-45e5-9e06-30caad274e68" target="_blank">link</a>) </li>
</ul>
<h4 id="instruments">Sentinel-3 Instruments:</h4>
The Sentinel-3 mission includes several key instruments, each serving distinct functions for Earth observation. Here's a list of the instruments along with a brief explanation of each:
<ol>
<li><b>Sea and Land Surface Temperature Radiometer (SLSTR): </b>
<ul>
<li><b>Function: </b>Measures global sea- and land-surface temperatures daily with an accuracy better than 0.3 K.</li>
<li><b>Description: </b>This dual-view (near-nadir and inclined) conical imaging radiometer builds on the heritage of the ENVISAT AATSR instrument. It provides accurate temperature measurements essential for climate monitoring and weather forecasting.
<ul>
<li>Includes two thermal infrared channels for active fire detection and fire radiative power measurement.</li>
<li>Supports Copernicus Emergency Response and Climate Services.</li>
<li>covering 9 spectral bands (550–12 000 nm), dual-view scan with swath widths of 1420 km (nadir) and 750 km (backwards), and a spatial resolution of 500 m for visible and near-infrared, and 1 km for thermal infrared channels.
<ul>
<li><b>S1 (0.555 μm):</b> Cloud screening, vegetation monitoring, and aerosol detection.</li>
<li><b>S2 (0.659 μm):</b> Normalized Difference Vegetation Index (NDVI), vegetation monitoring, and aerosol detection.</li>
<li><b>S3 (0.865 μm):</b> NDVI, cloud flagging, and pixel co-registration.</li>
<li><b>S4 (1.375 μm):</b> Cirrus detection over land.</li>
<li><b>S5 (1.61 μm):</b> Cloud clearing, ice, and snow detection.</li>
<li><b>S6 (2.25 μm):</b> Cloud clearing.</li>
<li><b>S7 (3.74 μm):</b> Surface temperature and active fire monitoring.</li>
<li><b>S8 (10.85 μm):</b> Surface temperature.</li>
<li><b>S9 (12.0 μm):</b> Surface temperature and emissivity.</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li><b>Ocean and Land Colour Instrument (OLCI): </b>
<ul>
<li><b>Function: </b>Captures detailed optical images to monitor ocean and land color, supporting studies of marine biology, water quality, and vegetation.</li>
<li><b>Description: </b>This push-broom imaging spectrometer is based on the heritage of the ENVISAT MERIS instrument. It collects data in multiple spectral bands, allowing for comprehensive analysis of oceanic and terrestrial ecosystems.
<ul>
<li>Features 21 distinct bands in the spectral region of (400 - 1040 nm).
<ul>
<li><b>Oa01 (400-412 nm):</b> Used for atmospheric correction.</li>
<li><b>Oa03 (442-454 nm):</b> Used for chlorophyll absorption peak</li>
<li><b>Oa08 (665-677 nm):</b> Used for chlorophyll absorption.</li>
<li><b>Oa17 (709-712 nm):</b> Used for detecting fluorescence from chlorophyll.</li>
<li><b>Oa21 (1020-1040 nm):</b> Used for atmospheric correction and water vapor content</li>
</ul>
</li>
<li>Swath width of 1270 km, overlapping with SLSTR swath.</li>
<li> covering 21 spectral bands (400–1020 nm) with a swath width of 1270 km and a spatial resolution of 300 m. </li>
</ul>
</li>
</ul>
</li>
<li><b>Microwave Radiometer (MWR): </b>
<ul>
<li><b>Function: </b>Provides wet atmosphere correction to enhance the accuracy of the altimeter's measurements.</li>
<li><b>Description: </b>This instrument measures the brightness temperature at specific microwave frequencies, helping to correct the signal delays caused by water vapor in the atmosphere. It features independent thermal control and cold redundancy for all subsystems to ensure reliability</li>
<ul>
<li>Operates in dual frequencies at 23.8 & 36.5 GHz
<ul>
<li><b>23.8 GHz:</b> Sensitive to water vapor in the atmosphere.</li>
<li><b>36.5 GHz:</b> Used for detecting liquid water content in the atmosphere, surface emissivity, and soil moisture.</li>
</ul>
</li>
<li>Supports the SAR altimeter.</li>
<li>Derives atmospheric correction and atmospheric column water vapour measurements.</li>
</ul>
</ul>
</li>
<li><b>Synthetic Aperture Radar (SAR) Radar Altimeter (SRAL): </b>
<ul>
<li><b>Function:</b> Measures sea surface height, wave height, and wind speed over the oceans. Provides accurate topography measurements over sea ice, ice sheets, rivers, and lakes.</li>
<li><b>Description: </b>SRAL operates in two frequency bands. This dual-frequency SAR altimeter builds on heritage from the ENVISAT RA-2, CryoSat SIRAL, and Jason-2/Poseidon-3 missions. It provides accurate and reliable altimetry measurements essential for ocean topography and sea state monitoring</li>
<ul>
<li><b>Ku-band (13.575 GHz):</b> Used for high-resolution altimetry over the ocean, coastal zones, sea ice, and inland waters.</li>
<li><b>C-band (5.41 GHz):</b> Used for reducing ionospheric delay effects and improving accuracy over different surface types .</li>
</ul>
</ul>
</li>
<li><b>Precise Orbit Determination (POD) Package: </b>
<ul>
<li><b>Components: </b>
<ul>
<li>Global Navigation Satellite Systems (GNSS) instrument.</li>
<li>Doppler Orbitography and Radiopositioning Integrated by Satellite (DORIS) instrument.</li>
<li>Laser retro-reflector (LRR).</li>
</ul>
</li>
<li><b>Function: </b>Ensures high-accuracy radial orbit data, which is crucial for precise altimetry measurements.</li>
<li><b>Description: </b>This package combines several technologies to provide precise orbital data, which is necessary for the accurate geolocation of the altimetry data.</li>
</ul>
</li>
</ol>
</section>
<section>
<h2 id="OCLI-module">OLCI module</h2>
The Ocean and Land Colour Instrument (OLCI) provides spectral information on the colour of the oceans. It is a <a href="https://earthobservatory.nasa.gov/features/RemoteSensing/remote_08.php" target="_blank">radiometer</a>, which measures light reflected by the Earths surfaces. This signal contains a rich wealth of information about the composition of ocean waters, the land surface, and atmosphere. EUMETSAT produces the marine products from the Sentinel-3 OLCI sensors. Ocean colour data provides a window in to the biological activity of the worlds oceans, as well as other oceanographic and coastal processes around sediments and anthropogenic impacts. Sentinel-3 OLCI data can also be used for atmospheric composition applications, including monitoring fires, aerosols, and dust. Current OLCI marine products contain some atmospheric composition related products, and further products are in development. Sentinel-3 OLCI land products are processed by the European Space Agency (ESA).
<p>For example, this data can be used to monitor global ocean primary production by <a href="https://oceanservice.noaa.gov/facts/phyto.html" target="_blank">phytoplankton</a>, the basis of nearly all life in our seas. <a href="https://oceancolor.gsfc.nasa.gov/" target="_blank">Ocean colour</a> data is also vital to understand <a href="https://climateknowledgeportal.worldbank.org/overview" target="_blank">climate change</a> — ocean colour is one of the <a href="climate-data.html" target="_blank">Essential Climate Variables</a> listed by the <a href="https://wmo.int/" target="_blank">World Meteorological Organization</a> to detect biological activity in the ocean’s surface layer. Phytoplankton take up carbon dioxide (CO\(_2\)) during photosynthesis, making them important carbon sinks. </p>
<ul>
<li>Ocean colour data can be used to monitor the annual global uptake of CO\(_2\) by phytoplankton on a global scale. Using this data we can study the wider Earth system, for instance the <a href="https://oceanservice.noaa.gov/facts/ninonina.html" target="_blank">El Niño/La Niña phenomena</a> and how these impacts the ocean ecosystem. Beyond climate, ocean colour data is also useful to look at more sporadic events.</li>
<li>OLCI data can be used track sediment transport, monitor coastal water quality and track and forecast harmful algal blooms that are a danger to humans, marine/freshwater life and aquaculture.</li>
</ul>
The global picture of ocean ecosystems provided by ocean colour data can guide sustainable marine resource management. Further information on the sensor and its data can be found at <a href="https://user.eumetsat.int/resources/user-guides/sentinel-3-olci-level-1-data-guide" target="_blank">http://olci.eumetsat.int</a> and in the <a href="https://eumetsatspace.atlassian.net/wiki/spaces/SEN3/overview" target="_blank">Sentinel-3 knowledge base</a>.
<br><br>
<h4 id="olci-spectrum">OLCI spectrum</h4>
The 21 wavelengths used to measure TOA radiance. The measured spectral response functions, which described the true shape of the wavebands, are available for both Sentinel-3A and -3B. The sampling period of the OLCI detectors (CCD’s) runs at 44ms, which correspond to a mean along-track distance of about 300m. Therefore, OLCI produces the Full Resolution (FR) product with a spatial resolution of approximately 300m. In addition, the Reduced Resolution (RR) product is generated with a spatial resolution of approximately 1.2km with 16 (nominally four by four) FR pixels averaged to create a RR pixel; the number of pixels used is varied in the across track direction, as a function of the across track pointing angle from the RR product pixel centre, so that spatial resolution degradation is minimised at the FOV edges.
<br><br>
The OLCI bands are available below in the table (taken from <a href="https://user.eumetsat.int/resources/user-guides/sentinel-3-olci-level-1-data-guide" target="_blank">OLCI website</a>).
<table>
<thead>
<tr>
<th>Band number</th>
<th>Central wavelength [nm]</th>
<th>Spectral width [nm]</th>
<th>Function</th>
</tr>
</thead>
<tbody>
<tr>
<td>Oa1</td>
<td>400.000</td>
<td>15</td>
<td>Atmospheric correction, improved water constituent retrieval</td>
</tr>
<tr>
<td>Oa2</td>
<td>412.500</td>
<td>10</td>
<td>Coloured Dissolved Organic Matter (CDOM) and detrital pigments</td>
</tr>
<tr>
<td>Oa3</td>
<td>442.500</td>
<td>10</td>
<td>Chlorophyll-a (Chl-a) absorption maximum</td>
</tr>
<tr>
<td>Oa4</td>
<td>490.000</td>
<td>10</td>
<td>High Chl-a and other photosynthetic pigments</td>
</tr>
<tr>
<td>Oa5</td>
<td>510.000</td>
<td>10</td>
<td>Chl-a, TSM, turbidity and red tides</td>
</tr>
<tr>
<td>Oa6</td>
<td>560.000</td>
<td>10</td>
<td>Chl-a reference (minimum)</td>
</tr>
<tr>
<td>Oa7</td>
<td>620.000</td>
<td>10</td>
<td>TSM</td>
</tr>
<tr>
<td>Oa8</td>
<td>665.000</td>
<td>10</td>
<td>Chl-a (2nd absorption maximum), TSM and CDOM</td>
</tr>
<tr>
<td>Oa9</td>
<td>673.750</td>
<td>7.5</td>
<td>Improved fluorescence retrieval</td>
</tr>
<tr>
<td>Oa10</td>
<td>681.250</td>
<td>7.5</td>
<td>Chl-a fluorescence peak</td>
</tr>
<tr>
<td>Oa11</td>
<td>708.750</td>
<td>10</td>
<td>Chl-a fluorescence baseline</td>
</tr>
<tr>
<td>Oa12</td>
<td>753.750</td>
<td>7.5</td>
<td>O2 (oxygen) absorption and clouds</td>
</tr>
<tr>
<td>Oa13</td>
<td>761.250</td>
<td>2.5</td>
<td>O2 absorption band and AC aerosol estimation</td>
</tr>
<tr>
<td>Oa14</td>
<td>764.375</td>
<td>3.75</td>
<td>AC</td>
</tr>
<tr>
<td>Oa15</td>
<td>767.500</td>
<td>2.5</td>
<td>O2 A-band absorption band used for cloud top pressure</td>
</tr>
<tr>
<td>Oa16</td>
<td>778.750</td>
<td>15</td>
<td>AC aerosol estimation</td>
</tr>
<tr>
<td>Oa17</td>
<td>865.000</td>
<td>20</td>
<td>AC aerosol estimation, clouds and pixel co-registration, in common with SLSTR</td>
</tr>
<tr>
<td>Oa18</td>
<td>885.000</td>
<td>10</td>
<td>Water vapour absorption reference band</td>
</tr>
<tr>
<td>Oa19</td>
<td>900.000</td>
<td>10</td>
<td>Water vapour absorption</td>
</tr>
<tr>
<td>Oa20</td>
<td>940.000</td>
<td>20</td>
<td>Water vapour absorption, linked to AC</td>
</tr>
<tr>
<td>Oa21</td>
<td>1020.000</td>
<td>40</td>
<td>AC aerosol estimation</td>
</tr>
</tbody>
</table>
<br><br>
<!----------------------------------->
<h4 id="baseline">OLCI processing baseline</h4>
The OLCI operational processor comprises three major processing levels: Level-0, Level-1 and Level-2 (for more details, please check the <a href="Satellite-data.html">link</a>):
<ul>
<li><b>Level 0:</b> This step processes raw data contained in instrument source packets. Level 0 products are internal products, and are not disseminated to users.</li>
<li><b>Level 1:</b> This step processes the level 0 data to geo-located and calibrated top of atmosphere radiances for each OLCI band.</li>
<li><b>Level 2:</b> This step processes the level 1 data to water leaving reflectances and bio-optical variables (eg chlorophyll-a concentration).</li>
</ul>
<figure>
<img src="assets/img/remote-sensing/olci-products-pdp.png" alt="" style="max-width: 30%; max-height: auto;">
<figcaption style="text-align: center;"><b></b> <a href="" target="_blank"></a>Schematic of the products generated by the OLCI processor at each level</figcaption>
</figure>
<div class="grey-box"><a href="https://user.eumetsat.int/resources/user-guides/sentinel-3-ocean-colour-level-2-data-guide" target="_blank">For more details on the Sentinel-3 ocean colour level 2 data guide</a>, please follow the original EUMESAT documentation.</div>
<p>The OLCI level 1B processor operates in three different modes, each creating specific types of products:</p>
<ol>
<li><b>Radiometric Calibration Mode:: </b>Processes calibration data to develop radiometric models (used to measure the intensity of light). The radiometric model is built using continuous in-flight calibration data. It includes: Radiometric Gain Coefficients: These are reference values that help convert raw data into accurate measurements, updated regularly to ensure accuracy. Long-term Evolution Model: This model tracks and adjusts for changes over time.</li>
<li><b>Spectral Calibration Mode: </b>Processes calibration data to develop spectral models (used to measure the light's wavelength).</li>
<li><b>Earth Observation Mode: </b>Produces detailed and location-specific Earth observation data at full and reduced resolutions.</li>
</ol>
<p></p>
<!------------------------------------>
<h4 id="olci-product">OLCI Product</h4>
<p>Data from all the Sentinel satellites operated under the <a href="https://www.copernicus.eu/en" target="_blank">European Commissions Copernicus Programme</a> are delievered in "<a href="https://user.eumetsat.int/resources/user-guides/sentinel-safe-format-guide" target="_blank">SAFE format</a>". The Sentinel-SAFE format is a specific variation of the Standard Archive Format for Europe (SAFE) format specification designed for the Sentinel satellite products. It is based on the XML Formatted Data Units (XFDU) standard under development by the <a href="https://public.ccsds.org/default.aspx" target="_blank">Consultative Committee for Space Data Systems (CCSDS)</a>. Sentinel-SAFE is a profile of XFDU, and it restricts the XFDU specifications for specific utilisation in the Earth Observation domain, providing semantics in the same domain to improve interoperability between ground segment facilities.</p>
<p>Each product package includes:</p>
<ul>
<li>a manifest file containing a metadata section and a data object section (an xml file).</li>
<li>measurement data files (NetCDF-4 format)</li>
<li>annotation data files, if defined (NetCDF-4 format)</li>
</ul>
<p>The product package can exist as a directory in a filesystem, zipped folder or tarball. The containing unit has a set file name that describes the product.</p>
<p>The Sentinel-SAFE product package file naming is based on a sequence of fields (for more details, please check <a href="https://user.eumetsat.int/resources/user-guides/sentinel-safe-format-guide" target="_blank">official documentation</a>):</p>
<figure>
<img src="assets/img/remote-sensing/sentinel-3-naming.png" alt="" style="max-width: 100%; max-height: auto;">
<figcaption style="text-align: center;"><b></b> <a href="" target="_blank"></a></figcaption>
</figure>
<div class="grey-box">
<p>For example, Sentinel-3 near real-time OLCI marine data at Level-2 product file looks like: <code>S3B_SL_2_AOD____20210803T215713_20210803T220308_20210804T002802_0355_055_229______MAR_O_NR_001</code>, where:
<figure>
<img src="assets/img/remote-sensing/sentinel-filename-structure.png" alt="" style="max-width: 90%; max-height: auto;">
<figcaption style="text-align: center;"><b></b> <a href="" target="_blank"></a></figcaption>
</figure>
1 would be:
<ul>
<li>S3B for Sentinel 3B</li>
<li>SL for SLSTR data source</li>
<li>L=2, for level 2 data</li>
<li>20210803T215713: Sensing start at 03 Aug 2021 at 21:37:13 hr. </li>
<li>20210803T220308: Sensing stops at 03 Aug 2021 at 22:03:08 hr. </li>
<li>20210804T002802: Product created at 04 Aug 2021 at 00:28:02 hr.</li>
<li>0355_055_229____ (17 character): 'Duration'_'cycle-number'_'relative orbit number'____</li>
<li>GGG–Product Generating Centre:
<ul>
<li>'MAR' for Marine Processing and Archiving Centre (EUMETSAT).</li>
<li>'LN3' for Land Surface Topography Mission Processing and Archiving Centre.</li>
<li>SVL = Svalbard Satellite Core Ground Station.</li>
</ul>
</li>
<li>O_NR_001 (format 'P_XX_NNN'): Eight characters to indicate the processing system. So here O=operational, NR for near real-time. Three letters/digits ('001') indicating the baseline collection. A significant change typically triggers a reprocessing and a new baseline collection.</li>
</ul>
</p>
</div>
<hr>
<br><br>
<!----------------------------------------->
<h2 id="olci-sent-analysis">OLCI Sentinel-3 data (analysis)</h2>
<h4 id="data-sentinel3">Accessing OLCI data via the EUMETSAT Data Store </h4>
<div class="alert alert-info"> <i class="bi bi-arrow-right-circle"></i> The notebooks are provided by EUMESAT during <font color="#F33365">Copernicus Marine Training programm 2023</font> and Authors are "<strong>Ben Loveday (EUMETSAT/Innoflair UG), Hayley Evers-King (EUMETSAT), Ana Ruescas (Brockmann Consult GmbH / University of Valencia)</strong>".<br>
</div>
In the next step, we will need data. We took three datasets from Sentinel-3 OLCI data (thanks to the data provided by EUMESAT).
<table border="1">
<tr>
<th>Product Description</th>
<th>Data Store collection ID</th>
<th>Product Navigator</th>
</tr>
<tr>
<td>Sentinel-3 OLCI level-1B full resolution</td>
<td>EO:EUM:DAT:0409</td>
<td><a href="https://navigator.eumetsat.int/product/EO:EUM:DAT:SENTINEL-3:OL_1_EFR___NTC?query=OLCI&filter=satellite__Sentinel-3&filter=instrument__OLCI&filter=processingLevel__Level%201%20Data&s=advanced" target="_blank">link</a></td>
</tr>
<tr>
<td>Sentinel-3 OLCI level-2 full resolution</td>
<td>EO:EUM:DAT:0407</td>
<td><a href="https://navigator.eumetsat.int/product/EO:EUM:DAT:SENTINEL-3:OL_2_WFR___NTC?query=OLCI&filter=satellite__Sentinel-3&filter=instrument__OLCI&filter=processingLevel__Level%202%20Data&s=advanced" target="_blank">link</a></td>
</tr>
<tr>
<td>Sentinel-3 OLCI level-2 reduced resolution</td>
<td>EO:EUM:DAT:0408</td>
<td><a href="https://navigator.eumetsat.int/product/EO:EUM:DAT:SENTINEL-3:OL_2_WRR___NTC?query=OLCI&filter=satellite__Sentinel-3&filter=instrument__OLCI&filter=processingLevel__Level%202%20Data&s=advanced" target="_blank">link</a></td>
</tr>
</table>
<p>The project directory is as follows:</p>
<pre class="language-css"><code>
EUMETSAT-courses/
│
├── env/
├── Sentinel-3_OLCI/
│ ├── 1_OLCI_introductory/
│ │ ├── products/
│ │ │ ├── S3A_OL_1_EFR____20210717T101015_20210717T101315_20210718T145224_0179_074_122_1980_MAR_O_NT_002.SEN3/
│ │ │ │ ├── geo_coordinates.nc
│ │ │ │ ├── Oa01_radiance.nc
│ │ │ └── S3A_OL_2_WFR____20210717T101015_20210717T101315_20210718T221347_0179_074_122_1980_MAR_O_NT_003.SEN3/
│ │ │ ├── chl_nn.nc
│ │ │ └── geo_coordinates.nc
│ │ ├── 1_1a_OLCI_data_access_Data_Store.ipynb
│ │ ├── 1_1b_OLCI_data_access_HDA.ipynb
│ │ ├── 1_2_OLCI_file_structure.ipynb
│ │ ├── 1_3_OLCI_coverage.ipynb
│ │ ├── 1_4_OLCI_bands_imagery.ipynb
│ │ ├── 1_5_OLCI_radiance_reflectance_spectra.ipynb
│ │ ├── 1_6_OLCI_CHL_comparison.ipynb
│ │ └── 1_7_OLCI_light_environment.ipynb
│ │
│ ├── 2_OLCI_advanced/
│ ├── frameworks/
│ │ └── config.ini
│ └── environment.yml (conda) or requirments.txt (with pip)
│── AUTHORS.txt
├── CHANGELOG
├── .gitignore
├── .gitmodules
├── README.md
└── LICENSE.txt
</code></pre>
<div class="box">
<p> In the Python ecosystem, managing packages and environments efficiently is crucial for maintaining a clean and reproducible workflow. Two of the most popular tools for these tasks are <code>Conda</code> and <code>pip</code>. While they both serve to manage packages, they have different strengths and use cases.</p>
<ol>
<li><strong>Conda: </strong>Conda is an open-source package and environment management system that originally catered to Python but now supports multiple languages. Conda operates seamlessly on Windows, macOS, and Linux, making it a versatile tool for developers working across different operating systems. Conda installs precompiled binary packages, which reduces the need for compiling code during installation. This feature is particularly useful for scientific computing packages that have complex dependencies. Conda automatically resolves dependencies, ensuring that all installed packages are compatible with each other. This minimizes the risk of dependency conflicts.
<ul>
<li><strong>Create a new environment: </strong>To construct the environment, in terminal/command prompt navigate to repository folder you want to create the environment, and use command to create the pytho environment using: <code>conda create -n myenv</code></li>
<li><strong>Activate an environment: </strong> <code>conda activate myenv</code></li>
<li><strong>Install a package: </strong> <code>conda install package_name</code></li>
<li><strong>Export an environment to a file: </strong> <code>conda env export > environment.yml</code></li>
<li><strong>Create an environment from a file: </strong> <code>conda env create -f environment.yml</code></li>
<li><strong></strong> <code></code></li>
</ul>
</li>
<li><strong>Pip: </strong> pip is the package installer for Python, and it connects directly to the Python Package Index (PyPI), where the majority of Python packages are hosted. pip is widely used in the Python community and is supported by almost all Python projects and libraries, ensuring broad compatibility. pip's simplicity and straightforward command structure make it an easy-to-use tool for managing Python packages.
<ul>
<li><strong>Set up virtual python environment: </strong>Similar to pip, following command can be used to create the python virtual enviroment:
<code>python -m venv env</code> and to activate it on:
<ul>
<li><strong>In Mac Os: </strong> <code>source env/bin/activate</code></li>
<li><strong>In windows using Command Prompt: </strong><code>env\Scripts\activate</code></li>
<li><strong>In windows using powershell: </strong><code>.\env\Scripts\Activate.ps1</code></li>
</ul>
</li>
<li><strong>PyPI Integration: </strong> pip connects to PyPI, giving you access to a vast repository of Python packages. This makes it easy to find and install the libraries you need. <code>pip install requests</code></li>
<li><strong>Install a package: </strong> <code>pip install package_name</code></li>
<li><strong>List installed packages: </strong><code>pip list</code></li>
<li><strong>Create a requirements file: </strong> <code>pip freeze > requirements.txt</code></li>
<li><strong>Install packages from a requirements file: </strong>pip supports requirements files (requirements.txt), which allow you to list all your project dependencies. This makes it easy to share and replicate environments. <code>pip install -r requirements.txt</code></li>
</ul>
</li>
</ol>
<p><strong>Conda vs. pip: Which Should You Use?</strong></p>
While both Conda and pip are valuable tools, they serve slightly different purposes and can often be used together:
<ul>
<li>Conda is best for creating isolated environments and managing complex dependencies, especially for data science and scientific computing projects.</li>
<li>pip is ideal for installing and managing Python packages from PyPI and is straightforward for general-purpose Python development.</li>
</ul>
<p><strong>Using Conda and pip Together:</strong> Many developers use Conda for environment management and pip for accessing the extensive range of packages on PyPI. Here’s a workflow combining both tools:</p>
<ul>
<li><strong>Create and activate a Conda environment: </strong>
<pre><code class="language-python">
conda create -n myenv
conda activate myenv
</code></pre>
</li>
<li><strong>Install packages using pip within the Conda environment: </strong>
<pre><code class="language-python">
pip install requests
</code></pre>
</li>
</ul>
</div>
<p>In the present case, to create the environment, run: <code>conda env create -f environment.yml</code>. This will create a Python environment called cmts_learn_olci. The environment won't be activated by default. To activate it, run: <code>conda activate cmts_learn_olci</code>.</p>
<ul>
<li><b>Step- 1: </b>Importing libraries
<pre><code class="language-python">
import configparser # a library that allows us to parse standard configuration files
import IPython # a library that helps us display video and HTML content
import os # a library that allows us access to basic operating system commands like making directories
import json # a library that helps us make JSON format files
import shutil # a library that allows us access to basic operating system commands like copy
import zipfile # a library that allows us to unzip zip-files.
import eumdac # a tool that helps us download via the eumetsat/data-store
</code></pre>
</li>
<li><b>Step- 2: </b>Sometimes we use configuration files to help us set some notebook parameters. The box below reads a configuration file to help us decide how large to make the videos displayed below
<pre><code class="language-python">
# Define the path to the configuration file
config_file_path = os.path.join(os.path.dirname(os.getcwd()), "frameworks", "config.ini")
# Check if the configuration file exists and read it if it does
if os.path.exists(config_file_path):
config.read(config_file_path)
</code></pre>
THis script check for configuration file and read if it exists in the current working directory.
<ul>
<li><code>os.getcwd()</code>: Returns the current working directory.</li>
<li><code>os.path.dirname</code>: Returns the directory name of the pathname.</li>
<li><code>os.path.join(path, *paths)</code>: Joins one or more path components intelligently.</li>
<li><code>os.path.exists(path)</code>: Returns True if the specified path exists, False otherwise.</li>
</ul>
For more details on os module, please check the <a href="https://arunp77.github.io/os-module.html" target="_blank">os-python module link</a>.
</li>
<li><b>Step- 3: </b>Creating a download directory for the data files:
<pre class="language-python"><code>
download_dir = os.path.join(os.getcwd(), "products")
os.makedirs(download_dir, exist_ok=True)
</code></pre>
</li>
<li><b>Step- 4: Data access: </b>The <a href="https://data.eumetsat.int" target="_blank">Data Store</a> is EUMETSAT's primary pull service for delivering data, including the ocean data available from Sentinel-3 and OLCI. Access to it is possible through a WebUI, and through a series of application programming interfaces (APIs). The Data Store supports browsing, searching and downloading data as well as subscription services. It also provides a link to the online version of the <a href="https://tailor.eumetsat.int/" target="_blank">EUMETSAT Data Tailor</a> for customisation. To access Sentinel-3 data from the <a href="https://data.eumetsat.int">EUMETSAT Data Store</a>, there are following methods that can be used:
<ul>
<li><a href="https://tailor.eumetsat.int/" target="_blank">EUMETSAT Data WebUI</a></li>
<li><a href="https://user.eumetsat.int/resources/user-guides/eumetsat-data-access-client-eumdac-guide" target="_blank"> EUMETSAT Data Access Client</a> (`<code>eumdac</code>`)</li>
</ul>
<div class="alert alert-info"> <i class="bi bi-arrow-right-circle"></i> <font color="#F33365">Other way of accessig the data from the EUMESAT using the REST API</font> are as follows:<br>
<ul>
<li><a href="https://user.eumetsat.int/api-definitions/data-store-opensearch-api" target="_blank">Data Store OpenSearch API</a></li>
<li><a href="https://user.eumetsat.int/api-definitions/data-store-rest-api" target="_blank">Data Store Browse API</a></li>
<li><a href="https://user.eumetsat.int/api-definitions/data-store-download-api" target="_blank">Data Store Download API</a></li>
</ul>
</div>
In order to allow us to download data from the Data Store via API, we need to provide our credentials. We can do this in two ways
<ul>
<li><b>Option-1: </b>either by creating a file called `<code>.eumdac_credentials</code>` in our home directory (recommended)</li>
<li><b>Option-2: </b>or by supplying our credentials directly in this script. </li>
</ul>
<div class="grey-box">
<p><strong>Option-1: </strong>creating `<code>.eumdac_credentials</code>` in our home directory. </p>
For most computer systems the home directory can be found at the path \user\username, /users/username, or /home/username depending on your operating system.
In this file we need to add the following information exactly as follows;
<pre class="language-python"><code>
{
"consumer_key": "your_consumer_key",
"consumer_secret": "your_consumer_secret"
}
</code></pre>
You must replace '<code>your_consumer_key</code>' and '<code>your_consumer_secret</code>' with the information you extract from <a href="https://api.eumetsat.int/api-key/" target="_blank">https://api.eumetsat.int/api-key/</a> You will need a <a href="https://eoportal.eumetsat.int/">EUMETSAT Earth Observation Portal account</a> to access this link, and in order to see the information you must click the "Show hidden fields" button at the bottom of the page.
<figure>
<img src="assets/img/remote-sensing/eumetsat-api.png" alt="" style="max-width: 60%; max-height: auto;">
<figcaption style="text-align: center;"><b></b> <a href="" target="_blank"></a>.</figcaption>
</figure>
<!-- <iframe width="560" height="315" src="https://www.youtube.com/embed/Wm33FhbZhF4" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> -->
<div style="display: flex; justify-content: center; align-items: center;">
<iframe width="560" height="315" src="https://www.youtube.com/embed/Wm33FhbZhF4" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
</div>
<pre class="language-python"><code>
# Get the home directory of the current user
home_directory = os.path.expanduser("~")
# Construct the full path to the '.eumdac_credentials' file
credentials_path = os.path.join(home_directory, '.eumdac_credentials')
# Print the full path
print(f"The full path to the '.eumdac_credentials' file is: {credentials_path}")
</code></pre>
<p>This code will generate the <code>.eumdac_credentials</code> in the home directory of computer system.</p>
<strong>Note:</strong> your key and secret are permanent, so you only need to do this once, but you should take care to never share them
Make sure to save the file without any kind of extension.
Once you have done this, you can read in your credentials using the commands in the following cell. These will be used to generate a time-limited token, which will refresh itself when it expires.
<p>Instead of saving the credentials into the home directory of the computer systems, we can also save it to a <code>.env/</code> folder which should be kept secret by adding the file name to '<code>.gitignore</code>' file. To create <code>.env</code>, use <code>mkdir -p .env</code>, then move the <code>.eumdac_credentials</code> file into the <code>.env</code> from the home or create a new <code>.eumdac_credentials</code> into the <code>.env</code>. Follwoing python code can be used to create it into <code>.env</code>:</p>
<pre class="language-python"><code>
import os
import json
import eumdac
# Define the path to the '.eumdac_credentials' file in the '.env' directory
project_directory = os.path.dirname(os.path.abspath(__file__)) # Get the directory of the current script
env_directory = os.path.join(project_directory, '.env')
credentials_path = os.path.join(env_directory, '.eumdac_credentials')
# Read the credentials from the file
with open(credentials_path) as json_file:
credentials = json.load(json_file)
token = eumdac.AccessToken((credentials['consumer_key'], credentials['consumer_secret']))
print(f"This token '{token}' expires {token.expiration}")
# Create data store object
datastore = eumdac.DataStore(token)
</code></pre>
<p><b>Option 2: provide credentials directly</b> You can provide your credentials directly as follows;
<pre class="language-css"><code>token = eumdac.AccessToken((consumer_key, consumer_secret))</code></pre>
</p>
<div class="alert alert-info"><b>Note:</b> this method is convenient in the short term, but is not really recommended as you have to put your key and secret in this notebook, and run the risk of accidentally sharing them. This method also requires you to authenticate on a notebook-by-notebook basis.</div>
</div>
Now we have a token, we can see what OLCI specific collections we have in the Data Store.
<pre class="language-python"><code>
datastore = eumdac.DataStore(token)
for collection_id in datastore.collections:
try:
if "OLCI" in collection_id.title and "EO:EUM:DAT:0" in str(collection_id):
print(f"Collection ID({collection_id}): {collection_id.title}")
except:
pass
</code></pre>
Since, I have used so many data products, I receive following output:
<pre class="language-python"><code>
Collection ID(EO:EUM:DAT:0410): OLCI Level 1B Reduced Resolution - Sentinel-3
Collection ID(EO:EUM:DAT:0556): OLCI Level 2 Ocean Colour Full Resolution (version BC003) - Sentinel-3 - Reprocessed
Collection ID(EO:EUM:DAT:0557): OLCI Level 2 Ocean Colour Reduced Resolution (version BC003) - Sentinel-3 - Reprocessed
Collection ID(EO:EUM:DAT:0407): OLCI Level 2 Ocean Colour Full Resolution - Sentinel-3
Collection ID(EO:EUM:DAT:0408): OLCI Level 2 Ocean Colour Reduced Resolution - Sentinel-3
Collection ID(EO:EUM:DAT:0409): OLCI Level 1B Full Resolution - Sentinel-3
Collection ID(EO:EUM:DAT:0577): OLCI Level 1B Full Resolution (version BC002) - Sentinel-3 - Reprocessed
Collection ID(EO:EUM:DAT:0578): OLCI Level 1B Reduced Resolution (version BC002) - Sentinel-3 - Reprocessed
</code></pre>
So, for;
<ul>
<li><b>OLCI Level 1B Full Resolution: </b> we want <code>collection_id</code>: <em><b>EO:EUM:DAT:0409</b></em>.</li>
<li><b>OLCI Level 2 Ocean Colour Reduced Resolution: </b> we want <code>collection_id</code>: <em><b>EO:EUM:DAT:0408</b></em></li>
<li><b>OLCI Level 2 Ocean Colour Full Resolution: </b> we want <code>collection_id</code>: <em><b>EO:EUM:DAT:0407</b></em></li>
</ul>
</li>
<li><b>Step- 5: The products we need for current OLCI data analytics: </b>
We need a number of OLCI level-1b and level-2 products. These products are shown below in a python 'list' called 'product list'. We will retrieve these products from the Data Store.
<pre class="language-python">
product_list = [
'S3A_OL_1_EFR____20210717T101015_20210717T101315_20210718T145224_0179_074_122_1980_MAR_O_NT_002.SEN3',
'S3A_OL_2_WRR____20210717T095732_20210717T104152_20210718T152419_2660_074_122______MAR_O_NT_003.SEN3',
'S3A_OL_2_WFR____20210717T101015_20210717T101315_20210718T221347_0179_074_122_1980_MAR_O_NT_003.SEN3'
]
</pre>
</li>
<li><b>Step- 6: Getting the data (Level 1B): </b>We can access the data using either collection id or product id:
<pre class="language-python"><code>
# With the level-1B `collection_id` to the `datastore` object to choose the correct collection.
collection_id = 'EO:EUM:DAT:0409'
selected_collection = datastore.get_collection(collection_id)
</code></pre>
<p>With the product name:</p>
<pre class="language-python"><code>
selected_product = datastore.get_product(product_id=product_list[0], collection_id=collection_id)
</code></pre>
<p>Once we have collection id and product names, we can download the data from the EUMESAT data store:</p>
<pre class="language-python"><code>
with selected_product.open() as fsrc, open(os.path.join(download_dir, fsrc.name), mode='wb') as fdst:
print(f'Downloading {fsrc.name}')
shutil.copyfileobj(fsrc, fdst)
print(f'Download of product {fsrc.name} finished.')
</code></pre>
<p> where:
<ul>
<li><code>os.path.join(download_dir, fsrc.name)</code>: Constructs the full path for the destination file by combining download_dir and the name of the source file (fsrc.name).</li>
<li><code>mode='wb'</code>: Opens the destination file in write-binary mode, which is necessary for copying binary data.</li>
<li><code>fdst</code>: A file-like object representing the destination file that is being written to.</li>
<li><code>shutil.copyfileobj(fsrc, fdst)</code>: Copies the content from the source file (fsrc) to the destination file (fdst). This function reads from fsrc and writes to fdst until the end of the source file is reached.</li>
</ul>
</p>
<p>Finally, we have downloaded the data in the <code>products/</code> folder. Output of the above code is:</p>
<pre class="language-python"><code>
Downloading S3A_OL_1_EFR____20210717T101015_20210717T101315_20210718T145224_0179_074_122_1980_MAR_O_NT_002.SEN3.zip
Download of product S3A_OL_1_EFR____20210717T101015_20210717T101315_20210718T145224_0179_074_122_1980_MAR_O_NT_002.SEN3.zip
finished.
</code></pre>
<p>The L1B level product is downloaded as a zip file, so lets unzip it and remove the zip.</p>
<pre class="language-python"><code>
with zipfile.ZipFile(fdst.name, 'r') as zip_ref:
for file in zip_ref.namelist():
if file.startswith(str(selected_product)):
zip_ref.extract(file, download_dir)
print(f'Unzipping of product {selected_product} finished.')
os.remove(fdst.name)
</code></pre>
<pre>> Unzipping of product S3A_OL_1_EFR____20210717T101015_20210717T101315_20210718T145224_0179_074_122_1980_MAR_O_NT_002.SEN3 finished.</pre>
</li>
<li><strong>Step- 7: Level 2 data products: </strong> We can now check the <b>products</b> directory to see that we have the level 1B file. Now, lets pull all the parts together and download both level 2 files in a single loop.
<pre class="language-python"><code>
collection_ids = ["EO:EUM:DAT:0408", "EO:EUM:DAT:0407"]
for product_id, collection_id in zip(product_list[1:],collection_ids):
print(f"Retrieving: {product_id}")
selected_collection = datastore.get_collection(collection_id)
selected_product = datastore.get_product(product_id=product_id, collection_id=collection_id)
with selected_product.open() as fsrc, open(os.path.join(download_dir, fsrc.name), mode='wb') as fdst:
print(f'Downloading {fsrc.name}.')
shutil.copyfileobj(fsrc, fdst)
print(f'Download of product {fsrc.name} finished.')
with zipfile.ZipFile(fdst.name, 'r') as zip_ref:
for file in zip_ref.namelist():
if file.startswith(str(selected_product)):
zip_ref.extract(file, download_dir)
print(f'Unzipping of product {fdst.name} finished.')
os.remove(fdst.name)
</code></pre>
This script downloads products from a datastore using specified collection IDs, extracts the relevant files from the downloaded zip archives, and then deletes the zip files. The steps ensure that only necessary files are extracted and stored in the download_dir.
</li>
</ul>
<!---------------------------------------------->
<h4 id="OCLI-query">Querying OLCI file structure (Level-1B)</h4>
To query the data, we will first need to provide the path to the data.
<pre class="language-python"><coce>
# selecting SAFE directory
SAFE_directory = os.path.join(os.getcwd(), 'products',
'S3A_OL_1_EFR____20210717T101015_20210717T101315_20210718T145224_0179_074_122_1980_MAR_O_NT_002.SEN3')
</coce></pre>
<p>Next, we'll create another variable that takes this path, and finds and adds on the names of the manifest file within the SAFE folder. For Sentinel-3, this manifest is always called <code>xfdumanifest.xml</code> and contains very useful information about the nature of the contents of the SAFE format product.</p>
<pre class="language-python"><code>
# selecting SAFE manifest
SAFE_manifest = glob.glob(os.path.join(SAFE_directory, 'xfd*.xml'))[0]
</code></pre>
Then, we'll read the manifest file in to a Python object format called a dictionary. This will make it easier for us to interact with the product in the next step.
</section>
<!-------Reference ------->
<section id="reference">
<h2>References</h2>
<ul>
<li><a href="https://learn.opengeoedu.de/en/fernerkundung" target="_blank">Remote Sensing, Notes by Opengeodu.</a></li>
<li><a href="https://www.dspguide.com/ch1.htm" target="_blank">The Scientist and Engineer's Guide to Digital Signal Processing By Steven W. Smith.</a></li>
<li><a href="https://www.jenkins.io/doc/tutorials/" target="_blank">Jenkins documentation</a></li>
<li><a href="https://sisu.ut.ee/imageprocessing/book/1#:~:text=Image%20processing%20is%20a%20method,features%20associated%20with%20that%20image." target="_blank">Image processing lecture.</a></li>
</ul>
</section>
<hr>
<div style="background-color: #f0f0f0; padding: 15px; border-radius: 5px;">
<h3>Some other interesting things to know:</h3>
<ul style="list-style-type: disc; margin-left: 30px;">
<li>Visit my website on <a href="sql-project.html">For Data, Big Data, Data-modeling, Datawarehouse, SQL, cloud-compute.</a></li>
<li>Visit my website on <a href="Data-engineering.html">Data engineering</a></li>
</ul>
</div>
<p></p>
<div class="navigation">
<a href="index.html#portfolio" class="clickable-box">
<span class="arrow-left">Portfolio section</span>
</a>
<a href="Remote-sensing-content.html" class="clickable-box">
<span class="arrow-right">Content</span>
</a>
</div>
</div>
</section><!-- End Portfolio Details Section -->
</main><!-- End #main -->
<!-- ======= Footer ======= -->
<footer id="footer">
<div class="container">
<div class="copyright">
© Copyright <strong><span>Arun</span></strong>
</div>
</div>
</footer><!-- End Footer -->
<a href="#" class="back-to-top d-flex align-items-center justify-content-center"><i class="bi bi-arrow-up-short"></i></a>
<!-- Vendor JS Files -->
<script src="assets/vendor/purecounter/purecounter_vanilla.js"></script>
<script src="assets/vendor/aos/aos.js"></script>
<script src="assets/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
<script src="assets/vendor/glightbox/js/glightbox.min.js"></script>
<script src="assets/vendor/isotope-layout/isotope.pkgd.min.js"></script>
<script src="assets/vendor/swiper/swiper-bundle.min.js"></script>
<script src="assets/vendor/typed.js/typed.umd.js"></script>
<script src="assets/vendor/waypoints/noframework.waypoints.js"></script>
<script src="assets/vendor/php-email-form/validate.js"></script>
<!-- Template Main JS File -->
<script src="assets/js/main.js"></script>
<script>
document.addEventListener("DOMContentLoaded", function () {
hljs.initHighlightingOnLoad();
});
</script>
<!-- Initialize highlight.js -->
<script>
document.addEventListener('DOMContentLoaded', (event) => {
document.querySelectorAll('pre code').forEach((block) => {
hljs.highlightBlock(block);
});
});
</script>
</body>
</html>