Fixup CSG reports to batch-send; Tweak services definition for better…

drivers/ma_reports/app/models/ma_reports/csg_engage/program.rb

@@ -10,5 +10,17 @@ class Program < GrdaWarehouseBase
     belongs_to :agency, class_name: 'MaReports::CsgEngage::Agency'
     has_many :program_reports, class_name: 'MaReports::CsgEngage::ProgramReport'
     has_many :program_mappings, class_name: 'MaReports::CsgEngage::ProgramMapping'
+
+    def households_scope
+      enrollments_scope.heads_of_households.order(:HouseholdID)
+    end
+
+    def enrollments_scope
+      GrdaWarehouse::Hud::Enrollment.joins(:project).where(project: { id: project_ids })
+    end
+
+    def project_ids
+      program_mappings.pluck(:project_id)
+    end
   end
 end

drivers/ma_reports/app/models/ma_reports/csg_engage/program_report.rb

@@ -28,13 +28,19 @@ def reset
     def run
       reset
 
-      data = MaReports::CsgEngage::ReportComponents::Report.new(program).serialize
       cleanup_last_report
-      result = MaReports::CsgEngage::Credential.first.post(data)
+
+      results = []
+
+      (0..(program.households_scope.distinct(:HouseholdID).count / batch_size)).each do |batch_index|
+        data = MaReports::CsgEngage::ReportComponents::Report.new(program, batch_size: batch_size, batch_index: batch_index).serialize
+        results << MaReports::CsgEngage::Credential.first.post(data)
+      end
+
       update(
         completed_at: Time.zone.now,
-        raw_result: result,
-        json_result: JSON.parse(result),
+        raw_result: results.join('\n'),
+        json_result: results.map { |res| JSON.parse(res) },
         imported_program_name: program.csg_engage_name,
         imported_import_keyword: program.csg_engage_import_keyword,
       )
@@ -70,5 +76,9 @@ def completed_without_response?
     def other_status_text
       return 'Completed without response' if completed_without_response?
     end
+
+    def batch_size
+      1000
+    end
   end
 end

drivers/ma_reports/app/models/ma_reports/csg_engage/report_components/household.rb

@@ -6,10 +6,11 @@
 
 module MaReports::CsgEngage::ReportComponents
   class Household < Base
-    attr_accessor :hoh_enrollment
+    attr_accessor :hoh_enrollment, :enrollments
 
-    def initialize(hoh_enrollment)
+    def initialize(hoh_enrollment, enrollments)
       @hoh_enrollment = hoh_enrollment
+      @enrollments = enrollments
     end
 
     field('Household Identifier') do
@@ -46,7 +47,7 @@ def initialize(hoh_enrollment)
       field('Extra-Sensitive')
       field('Farmer')
       field('FoodStamps') do
-        boolean_string(@enrollments_scope.any? { |enrollment| enrollment.income_benefits.max_by(&:information_date)&.SNAP == 1 })
+        boolean_string(enrollments.any? { |enrollment| enrollment.income_benefits.max_by(&:information_date)&.SNAP == 1 })
       end
       field('Household Type')
       field('Housing Subsidy Type') do
@@ -70,7 +71,7 @@ def initialize(hoh_enrollment)
       field('Housing Type')
       field('HousingType', method: :housing_type_2)
       field('MigrantFarmer')
-      field('Number in House') { enrollments_scope.count.to_s }
+      field('Number in House') { enrollments.size.to_s }
       field('SeasonalFarmer')
     end
 
@@ -95,7 +96,7 @@ def initialize(hoh_enrollment)
     field('Household Members') do
       result = []
       number = 1
-      enrollments_scope.order(:personal_id).find_each do |enrollment|
+      enrollments.each do |enrollment|
         result << MaReports::CsgEngage::ReportComponents::HouseholdMember.new(enrollment, number)
         number += 1
       end
@@ -104,12 +105,12 @@ def initialize(hoh_enrollment)
 
     private
 
-    def enrollments_scope
-      @enrollments_scope ||= GrdaWarehouse::Hud::Enrollment.where(
-        project_id: hoh_enrollment.project_id,
-        household_id: hoh_enrollment.household_id,
-      ).preload(:income_benefits, :services)
-    end
+    # def enrollments_scope
+    #   @enrollments_scope ||= GrdaWarehouse::Hud::Enrollment.where(
+    #     project_id: hoh_enrollment.project_id,
+    #     household_id: hoh_enrollment.household_id,
+    #   ).preload(:income_benefits, :services)
+    # end
 
     def coc
       @coc ||= hoh_enrollment.project.project_cocs.first

drivers/ma_reports/app/models/ma_reports/csg_engage/report_components/household_member.rb

@@ -114,12 +114,10 @@ def initialize(enrollment, number)
     # field('Expense')
 
     field('Services') do
-      services = enrollment.services.order(DateProvided: :desc).limit(1)
+      latest_service = enrollment.services.max_by(&:DateProvided)
 
-      if services.present?
-        services.map do |service|
-          MaReports::CsgEngage::ReportComponents::Service.new(service)
-        end
+      if latest_service.present?
+        [MaReports::CsgEngage::ReportComponents::Service.new(latest_service)]
       else
         [
           {
@@ -215,7 +213,7 @@ def race_fields
     end
 
     def latest_income_benefit
-      @latest_income_benefit ||= enrollment.income_benefits.order(:information_date).last
+      @latest_income_benefit ||= enrollment.income_benefits.max_by(&:information_date)
     end
   end
 end

drivers/ma_reports/app/models/ma_reports/csg_engage/report_components/income.rb

@@ -9,7 +9,7 @@ class Income < Base
     attr_accessor :amount, :description, :income_source, :payer_name
 
     def initialize(amount: 0, description: nil, income_source: nil, payer_name: nil)
-      @amount = ActiveSupport::NumberHelper.number_to_delimited(amount)
+      @amount = amount
       @description = description
       @income_source = income_source
       @payer_name = payer_name

drivers/ma_reports/app/models/ma_reports/csg_engage/report_components/program.rb

@@ -6,20 +6,23 @@
 
 module MaReports::CsgEngage::ReportComponents
   class Program < Base
-    attr_accessor :program
+    attr_accessor :program, :batch_size, :batch_index
 
-    def initialize(program)
+    def initialize(program, batch_size: 1000, batch_index: 0)
       @program = program
       @now = DateTime.current
+      @batch_size = batch_size
+      @batch_index = batch_index
     end
 
     field('Program Name') { program.csg_engage_name }
     field('Import Keyword') { program.csg_engage_import_keyword }
 
     field('Households') do
       result = []
-      households_scope.find_each do |enrollment|
-        result << MaReports::CsgEngage::ReportComponents::Household.new(enrollment)
+      enrollments_index = enrollments.group_by(&:HouseholdID)
+      households_scope.find_each do |hoh_enrollment|
+        result << MaReports::CsgEngage::ReportComponents::Household.new(hoh_enrollment, enrollments_index[hoh_enrollment.HouseholdID])
       end
       result
     end
@@ -31,7 +34,14 @@ def project_ids
     end
 
     def households_scope
-      GrdaWarehouse::Hud::Enrollment.joins(:project).where(project: { id: project_ids }).heads_of_households.preload(project: [:project_cocs])
+      hh_ids = program.households_scope.limit(batch_size).offset(batch_size * batch_index).distinct(:HouseholdID).pluck(:HouseholdID)
+      program.households_scope.where(HouseholdID: hh_ids).preload(project: [:project_cocs])
+    end
+
+    def enrollments
+      hh_ids = households_scope.pluck(:HouseholdID)
+      project_ids = households_scope.pluck(:ProjectID)
+      program.enrollments_scope.where(HouseholdID: hh_ids, ProjectID: project_ids).preload(:client, :income_benefits, :services, :exit)
     end
   end
 end

drivers/ma_reports/app/models/ma_reports/csg_engage/report_components/report.rb

@@ -6,18 +6,20 @@
 
 module MaReports::CsgEngage::ReportComponents
   class Report < Base
-    attr_accessor :program
+    attr_accessor :program, :batch_size, :batch_index
 
-    def initialize(program)
+    def initialize(program, batch_size: 1000, batch_index: 0)
       @program = program
+      @batch_size = batch_size
+      @batch_index = batch_index
     end
 
     field('AgencyID') { program.agency.csg_engage_agency_id }
     field('Data Type') { 'Households' }
     field('Action') { 'Import' }
 
     field('Programs') do
-      [MaReports::CsgEngage::ReportComponents::Program.new(program)]
+      [MaReports::CsgEngage::ReportComponents::Program.new(program, batch_size: batch_size, batch_index: batch_index)]
     end
   end
 end

drivers/ma_reports/app/models/ma_reports/csg_engage/report_components/service.rb

@@ -34,7 +34,7 @@ def initialize(service)
       field('ServiceDateTimeBegin') { service.DateProvided&.strftime('%m/%d/%Y') }
       field('ServiceDateTimeEnd') { nil }
       field('ServiceProvided') do
-        HudUtility2024.service_type_provided(service.Recordtype, service.TypeProvided) || 'Unknown Service Type'
+        HudUtility2024.service_type_provided(service.Recordtype, service.TypeProvided) || 'Service Provided'
       rescue StandardError
         'Unknown Service Type'
       end

drivers/ma_reports/spec/models/ma_reports/csg_engage/report_components/report_spec.rb

@@ -156,7 +156,7 @@
           include(
             *MaReports::CsgEngage::ReportComponents::HouseholdMember::INCOME_MAPPINGS.map do |_field, amount_field, attrs|
               include(
-                'Amount' => (income_benefit.send(amount_field) * 12.0).round.to_s,
+                'Amount' => (income_benefit.send(amount_field) * 12.0).round,
                 'Description' => attrs[:description],
                 'IncomeSource' => attrs[:income_source],
               )

drivers/ma_reports/spec/models/ma_reports/csg_engage/report_spec.rb

@@ -64,7 +64,7 @@
         pr = report.program_reports.find_by(program_id: p.id)
         expect(pr).to have_attributes(
           raw_result: '{"ok":true}',
-          json_result: { 'ok' => true },
+          json_result: [{ 'ok' => true }],
           error_data: nil,
           warning_data: nil,
           started_at: be_present,