Exporting daily climate values in GEE for points over wide date range

Question

I am new to using Google Earth Engine, but trying to extract climate variables for a wide date range on a large dataset.

Example data can be found here: https://code.earthengine.google.com/?asset=projects/ee-marghughes052/assets/Help_Files/GEE_Example_data

In case the asset cannot be accessed the data can also be found here: https://drive.google.com/drive/folders/1dL6v42Z8PrfJC1IBLU704nBiVxuJKCFi?usp=sharing

In this data I have a point location that was taken on a specific 'DateTime' on the 'Date' by a person throughout the province of BC, Canada. There are some points taken by the same person ('ID') at multiple locations on the same Date. This was done for over 10 years, but I have filtered to just work with one year for now. I am trying to extract precipitation (prcp), max temperature (tmax), and snow water equivalent (swe) from the following dataset: NASA/ORNL/DAYMET_V4.

Currently my logic is to split my points by year, and then go through and extract the values for max temperature at each point and date in that one year, but I have not been able to get this to work. Furthermore, if there is a way I can do this for my entire date range (2010-2024) without it taking a really long time to compute that would be ideal.

// Load the telemetry points feature collection
var pts = ee.FeatureCollection('projects/ee-marghughes052/assets/Help_Files/GEE_Example_data');

// Function to buffer points with a given radius and optionally return bounds
function bufferPoints(radius, bounds) {
  return function(pt) {
    pt = ee.Feature(pt);
    return bounds ? pt.buffer(radius).bounds() : pt.buffer(radius);
  };
}

// Map buffer function to telemetry points to create buffered regions
//var ptsBuffered = pts.map(bufferPoints(30, true));
// when I try to buffer the points I get the following error: Error: Execution failed; out of memory. (Error code: 8)
// therefore working with the un-buffered points.
print(pts.limit(10), 'Locations')

// Load the Daymet maximum temperature dataset
//filter for the year we are currently working with 
var dataset = ee.ImageCollection('NASA/ORNL/DAYMET_V4')
                  .filter(ee.Filter.date('2011-01-01', '2011-12-31'));
var maximumTemperature = dataset.select('tmax');

// Define a projection based on the first image in the collection
var proj = maximumTemperature.first().projection();

// Function to calculate zonal statistics (median in this case)
function zonalStats(ic, fc, params) {
  var _params = {
    reducer: ee.Reducer.median(),
    scale: 500,
    crs: proj,
    bands: null,
    bandsRename: null,
    imgProps: null,
    imgPropsRename: null,
    datetimeName: 'datetime',
    datetimeFormat: 'YYYY-MM-dd HH:mm:ss'
  };

  if (params) {
    for (var param in params) {
      _params[param] = params[param] || _params[param];
    }
  }

  var imgRep = ic.first();
  var nonSystemImgProps = ee.Feature(null)
    .copyProperties(imgRep).propertyNames();
  if (!_params.bands) _params.bands = imgRep.bandNames();
  if (!_params.bandsRename) _params.bandsRename = _params.bands;
  if (!_params.imgProps) _params.imgProps = nonSystemImgProps;
  if (!_params.imgPropsRename) _params.imgPropsRename = _params.imgProps;

  var results = ic.map(function(img) {
    img = ee.Image(img.select(_params.bands, _params.bandsRename))
      .set(_params.datetimeName, img.date().format(_params.datetimeFormat))
      .set('timestamp', img.get('system:time_start'));

    var propsFrom = ee.List(_params.imgProps)
      .cat(ee.List([_params.datetimeName, 'timestamp']));
    var propsTo = ee.List(_params.imgPropsRename)
      .cat(ee.List([_params.datetimeName, 'timestamp']));
    var imgProps = img.toDictionary(propsFrom).rename(propsFrom, propsTo);

    var fcSub = fc.filterBounds(img.geometry());

    return img.reduceRegions({
      collection: fcSub,
      reducer: _params.reducer,
      scale: _params.scale,
      crs: _params.crs
    })
    .map(function(f) {
      return f.set(imgProps);
    });
  }).flatten().filter(ee.Filter.notNull(_params.bandsRename));

  return results;
}

// Parameters for zonal statistics function
var params = {
  reducer: ee.Reducer.median(),
  scale: 500,
  crs: proj,
  bands: ['tmax'],
  bandsRename: ['medianTemp'],
  datetimeName: 'date',
  datetimeFormat: 'YYYY-MM-dd'
};

// Extract zonal statistics per point per image
var ptsStats = zonalStats(maximumTemperature, pts, params);

// Print the first 50 results to check
print(ptsStats.limit(50),'zonal stats results');

// Display the maximum temperature layer on the map
var maximumTemperatureVis = {
  min: -40.0,
  max: 30.0,
  palette: ['1621A2', 'white', 'cyan', 'green', 'yellow', 'orange', 'red'],
};
Map.addLayer(maximumTemperature, maximumTemperatureVis, 'Maximum Temperature');

// Add points
Map.addLayer(pts,{},'telem points');

// If your browser times out, try exporting the results. It's likely that point feature collections that
// cover a large area or contain many points (point-image observations) will need to be exported as a batch 
// task by either exporting the final feature collection as an asset or as a CSV/Shapefile/GeoJSON to Google 
// Drive or GCS.
/*Export.table.toAsset({
  collection: ptsStats,
  description: 'MedianDailyMaxTemperature',
  assetId: 'median_daily_temps'
});*/
Export.table.toDrive({
  collection: ptsStats,
  folder: 'earth_engine_data',
  description: 'median_daily_temp_2011',
  fileFormat: 'CSV'
})

What I am trying to do with this code:

1. Filter the image to the year I am currently working on
2. Extract the max temperature that corresponds to the 'Date' that the location point was taken on
3. Export a .csv file that contains the following columns ID, Date (point was taken on), MaxTemp (from image collection), ImgDate (Date the temp value taken from). My hope is to have one row for each point location. Currently when I export to my Google Drive with the code above I get a blank .csv file.
4. Eventually I would like to loop through all years extracting the data from the 'swe' band and 'prcp' band as well without this taking a large amount of computation time.

Any advice/direction to resources? I know there are similar questions out there, but I can't seem to get those solutions to work for my data.

Answer 1

You have a lot of points (30,162) and you should work with Image and Feature Collections for guaranteeing side server operation. I tried out the following code with a sample of 1000 features of your Feature Collection for retrieving only Tmax for each point.

// Load the telemetry points feature collection
var pts = ee.FeatureCollection('projects/ee-marghughes052/assets/Help_Files/GEE_Example_data');

Map.addLayer(pts);
Map.centerObject(pts);

// Function to buffer points with a given radius and optionally return bounds
function bufferPoints(radius, bounds) {
  return function(pt) {
    pt = ee.Feature(pt);
    return bounds ? pt.buffer(radius).bounds() : pt.buffer(radius);
  };
}

// Map buffer function to telemetry points to create buffered regions
//var ptsBuffered = pts.map(bufferPoints(30, true));
// when I try to buffer the points I get the following error: Error: Execution failed; out of memory. (Error code: 8)
// therefore working with the un-buffered points.

print('Locations', pts.size());

// Load the Daymet maximum temperature dataset
//filter for the year we are currently working with 
var dataset = ee.ImageCollection('NASA/ORNL/DAYMET_V4')
                  .filter(ee.Filter.date('2011-01-01', '2012-01-01'));

var maximumTemperature = dataset.select('tmax');

pts = pts.limit(1000);

print(pts.limit(10));

var maxTemp = pts.map(function (ele){
  
  var pt = ee.Feature(ele).geometry();
  
  var tempVal = maximumTemperature.map(function(image) {
  
  var value_temp = ee.Number(ee.Image(image)
    .reduceRegion(ee.Reducer.first(), pt)
    .get('tmax'))
    .format('%.2f');
  
    return ee.Feature(null).set('tmax', value_temp);
  
  });
  
  return tempVal;
  
});

//print(maxTemp);

// Export features, specifying corresponding names.
Export.table.toDrive(maxTemp.flatten(),
  "export_temp", //my task
  "GEE_Folder", //my export folder
  "tmax_dayly",  //file name
  "CSV"
);

After running the task for 35 minutes, I got the CSV file of following link:

https://drive.google.com/file/d/1LXQHQWpgJMnvHGF59d05Lifgnj0ezUJg/view?usp=sharing

This file has 365,001 records with 365-366 values (one per day) for each feature of the sample (1000 features) of your Feature Collection. In the following picture it can be observed the transition of the last day of 2011 (2011-12-31) for the first feature (id 00000000000000000142) and the first day of 2011 (2011-01-01) for the second feature (id 000000000000000002af).

In the following picture it can be corroborated that ids are correct. The framed area in the Map Canvas corresponds to all 30,162 features included in your Feature Collection. Each point can be individualized with the corresponding Zoom In.

Editing Note:

Following code:

https://code.earthengine.google.com/4a0179bc23b4a9e0ae60fe6e880960e6

gets just the values for the date that someone was out to take that GPS location, which minimize the amount of data (tmax, prcp, swe, long, lat) you need to export (only 30,162 in this example sample).

The key was to put the filter date inside the function. You had it outside the function.

var climVals = pts.map(function (ele){
  
  var pt = ee.Feature(ele).geometry();
  var date = ee.Date(ee.Feature(ele).get('Date'));
  
  var date2 = ee.Date(date).format().slice(0,10);
  
  var lat = ee.Feature(ele).get('Latitude');
  
  var long = ee.Feature(ele).get('Longitude');
  
  var image = dataset.select('tmax', 'prcp', 'swe')
                     .filter(ee.Filter.date(date, ee.Date(date).advance(1, 'day')))
                     .first();
  
  var value_temp = ee.Number(ee.Image(image).reduceRegion(ee.Reducer.first(), pt)
                                            .get('tmax'))
                                            .format('%.2f');
                                            
  var value_prcp = ee.Number(ee.Image(image).reduceRegion(ee.Reducer.first(), pt)
                                            .get('prcp'))
                                            .format('%.2f');
  
  var value_swe = ee.Number(ee.Image(image).reduceRegion(ee.Reducer.first(), pt)
                                            .get('swe'))
                                            .format('%.2f');

  
  return ee.Feature(null).set('tmax', value_temp)
                         .set('prcp', value_prcp)
                         .set('swe', value_swe)
                         .set('long', long)
                         .set('lat', lat)
                         .set('date', date2);
  
});

The link to the CSV file (obtained in < 1 minute) is:

https://drive.google.com/file/d/1f6S2Xm6Fz_3ZIN7XIL5jnLV3smwo-GnI/view?usp=sharing

One extract of this file can be observed in following picture: