refs/refs/FhirToCdm/CdmPersonBuilder.cs

   1using org.ohdsi.cdm.framework.common.Base;
   2using org.ohdsi.cdm.framework.common.Builder;
   3using org.ohdsi.cdm.framework.common.Enums;
   4using org.ohdsi.cdm.framework.common.Extensions;
   5using org.ohdsi.cdm.framework.common.Helpers;
   6using org.ohdsi.cdm.framework.common.Lookups;
   7using org.ohdsi.cdm.framework.common.Omop;
   8using System;
   9using System.Collections.Concurrent;
  10using System.Collections.Generic;
  11using System.Linq;
  12using System.Threading.Tasks;
  13
  14namespace FHIRtoCDM
  15{
  16    public class CdmPersonBuilder : IPersonBuilder
  17    {
  18        #region Variables
  19
  20        private readonly Dictionary<long, VisitOccurrence> _rawVisits = new Dictionary<long, VisitOccurrence>();
  21        private readonly Dictionary<long, List<VisitDetail>> _visitDetails = new Dictionary<long, List<VisitDetail>>();
  22
  23        protected ChunkData ChunkData;
  24        protected KeyMasterOffsetManager Offset;
  25        protected IVocabulary Vocabulary;
  26        public List<EraEntity> ObservationPeriodsRaw = new List<EraEntity>();
  27        protected List<Person> PersonRecords = new List<Person>();
  28        protected List<Death> DeathRecords = new List<Death>();
  29        protected List<Cohort> CohortRecords = new List<Cohort>();
  30
  31        protected List<PayerPlanPeriod>
  32            PayerPlanPeriodsRaw = new List<PayerPlanPeriod>();
  33
  34        protected List<ConditionOccurrence> ConditionOccurrencesRaw =
  35            new List<ConditionOccurrence>();
  36
  37        protected List<DrugExposure> DrugExposuresRaw = new List<DrugExposure>();
  38
  39        protected List<ProcedureOccurrence> ProcedureOccurrencesRaw =
  40            new List<ProcedureOccurrence>();
  41
  42        protected List<Observation> ObservationsRaw = new List<Observation>();
  43        protected List<Measurement> MeasurementsRaw = new List<Measurement>();
  44
  45        protected List<VisitOccurrence>
  46            VisitOccurrencesRaw = new List<VisitOccurrence>();
  47
  48        protected List<VisitDetail>
  49            VisitDetailsRaw = new List<VisitDetail>();
  50
  51        protected List<VisitCost> VisitCostsRaw = new List<VisitCost>();
  52        protected List<DeviceExposure> DeviceExposureRaw = new List<DeviceExposure>();
  53        protected List<DeviceCost> DeviceCostRaw = new List<DeviceCost>();
  54
  55        protected List<DrugExposure> DrugForEra = new List<DrugExposure>();
  56        protected List<ConditionOccurrence> ConditionForEra = new List<ConditionOccurrence>();
  57
  58        protected List<Note> NoteRecords = new List<Note>();
  59
  60        #endregion
  61
  62
  63        #region Methods
  64
  65        public bool AddCost(long eventId, IEntity entity, Func<ICostV5, Cost> createCost)
  66        {
  67            // TMP
  68            var costDataExists = false;
  69            if (entity == null) return false;
  70
  71            switch (entity.GeEntityType())
  72            {
  73                case EntityType.DrugExposure:
  74                    {
  75                        var de = (DrugExposure)entity;
  76
  77                        if (de.DrugCost != null)
  78                        {
  79                            costDataExists |= AddCost(eventId, createCost, de, de.DrugCost);
  80                        }
  81
  82                        de.DrugCost = null;
  83                        break;
  84                    }
  85
  86                case EntityType.ProcedureOccurrence:
  87                    {
  88                        var p = (ProcedureOccurrence)entity;
  89                        if (p.ProcedureCosts != null && p.ProcedureCosts.Count > 0)
  90                        {
  91                            foreach (var pc in p.ProcedureCosts)
  92                            {
  93                                costDataExists |= AddCost(eventId, createCost, p, pc);
  94                            }
  95
  96                            p.ProcedureCosts.Clear();
  97                            p.ProcedureCosts = null;
  98                        }
  99                        break;
 100                    }
 101
 102
 103                case EntityType.Observation:
 104                    {
 105                        var o = (Observation)entity;
 106                        if (o.ObservationCost != null && o.ObservationCost.Count > 0)
 107                        {
 108                            foreach (var oc in o.ObservationCost)
 109                            {
 110                                costDataExists |= AddCost(eventId, createCost, o, oc);
 111                            }
 112                            o.ObservationCost.Clear();
 113                            o.ObservationCost = null;
 114                        }
 115                        break;
 116                    }
 117
 118                case EntityType.VisitOccurrence:
 119                    {
 120                        var vo = (VisitOccurrence)entity;
 121                        if (vo.VisitCosts != null && vo.VisitCosts.Count > 0)
 122                        {
 123                            foreach (var vc in vo.VisitCosts)
 124                            {
 125                                costDataExists |= AddCost(eventId, createCost, vo, vc);
 126                            }
 127                            vo.VisitCosts.Clear();
 128                            vo.VisitCosts = null;
 129                        }
 130                        break;
 131                    }
 132
 133                case EntityType.Measurement:
 134                    {
 135                        var m = (Measurement)entity;
 136                        if (m.MeasurementCost != null && m.MeasurementCost.Count > 0)
 137                        {
 138                            foreach (var mc in m.MeasurementCost)
 139                            {
 140                                costDataExists |= AddCost(eventId, createCost, m, mc);
 141                            }
 142                            m.MeasurementCost.Clear();
 143                            m.MeasurementCost = null;
 144                        }
 145                        break;
 146                    }
 147
 148                case EntityType.DeviceExposure:
 149                    {
 150                        var d = (DeviceExposure)entity;
 151                        if (d.DeviceCosts != null && d.DeviceCosts.Count > 0)
 152                        {
 153                            foreach (var dc in d.DeviceCosts)
 154                            {
 155                                costDataExists |= AddCost(eventId, createCost, d, dc);
 156                            }
 157
 158                            d.DeviceCosts.Clear();
 159                            d.DeviceCosts = null;
 160                        }
 161                        break;
 162                    }
 163            }
 164
 165            return costDataExists;
 166        }
 167
 168        public void Reset()
 169        {
 170            //ChunkData.Clean();
 171
 172            Vocabulary = null;
 173            ObservationPeriodsRaw.Clear();
 174            ObservationPeriodsRaw = null;
 175
 176            PersonRecords.Clear();
 177            PersonRecords = null;
 178            DeathRecords.Clear();
 179            DeathRecords = null;
 180            CohortRecords.Clear();
 181            CohortRecords = null;
 182
 183            PayerPlanPeriodsRaw.Clear();
 184            PayerPlanPeriodsRaw = null;
 185
 186            ConditionOccurrencesRaw.Clear();
 187            ConditionOccurrencesRaw = null;
 188
 189            DrugExposuresRaw.Clear();
 190            DrugExposuresRaw = null;
 191
 192            ProcedureOccurrencesRaw.Clear();
 193            ProcedureOccurrencesRaw = null;
 194
 195            ObservationsRaw.Clear();
 196            ObservationsRaw = null;
 197
 198            MeasurementsRaw.Clear();
 199            MeasurementsRaw = null;
 200
 201            VisitOccurrencesRaw.Clear();
 202            VisitOccurrencesRaw = null;
 203
 204            VisitDetailsRaw.Clear();
 205            VisitDetailsRaw = null;
 206
 207            VisitCostsRaw.Clear();
 208            VisitCostsRaw = null;
 209
 210            DeviceExposureRaw.Clear();
 211            DeviceExposureRaw = null;
 212
 213            DeviceCostRaw.Clear();
 214            DeviceCostRaw = null;
 215
 216            DrugForEra.Clear();
 217            DrugForEra = null;
 218
 219            ConditionForEra.Clear();
 220            ConditionForEra = null;
 221
 222            NoteRecords.Clear();
 223            NoteRecords = null;
 224        }
 225
 226        private bool AddCost(long eventId, Func<ICostV5, Cost> createCost, IEntity entity, ICostV5 entityCost)
 227        {
 228            if (entityCost == null) return false;
 229
 230            var cost = createCost(entityCost);
 231            cost.CostId = Offset.GetKeyOffset(entityCost.PersonId).VisitCostId;
 232            cost.EventId = eventId;
 233
 234            return ChunkData.AddCostData(cost);
 235        }
 236        public ChunkData Result => ChunkData;
 237
 238        public bool Complete { get; set; }
 239
 240
 241        public void AddChildData(ProcedureOccurrence parent, ProcedureCost pc)
 242        {
 243            parent.ProcedureCosts = new List<ProcedureCost> { pc };
 244        }
 245
 246        public void AddChildData(DrugExposure parent, DrugCost child)
 247        {
 248            parent.DrugCost = child;
 249        }
 250
 251        public void AddChildData(DeviceExposure parent, DeviceCost child)
 252        {
 253            parent.DeviceCosts = new List<DeviceCost> { child };
 254        }
 255
 256        public void AddChildData(Measurement parent, MeasurementCost child)
 257        {
 258            parent.MeasurementCost = new List<MeasurementCost> { child };
 259        }
 260
 261        public void AddChildData(Observation parent, ObservationCost child)
 262        {
 263            parent.ObservationCost = new List<ObservationCost> { child };
 264        }
 265
 266        public void AddChildData(VisitOccurrence parent, VisitCost child)
 267        {
 268            parent.VisitCosts = new List<VisitCost> { child };
 269        }
 270
 271        public void AddNote(Note data)
 272        {
 273            NoteRecords.Add(data);
 274        }
 275
 276        /// <summary>
 277        /// Add raw entities to builder for further build
 278        /// </summary>
 279        /// <param name="data">raw entity</param>
 280        public void AddData(IEntity data)
 281        {
 282            switch (data.GeEntityType())
 283            {
 284                case EntityType.Person:
 285                    {
 286                        AddEntity((Person)data, PersonRecords);
 287                        break;
 288                    }
 289
 290                case EntityType.Death:
 291                    {
 292                        AddEntity((Death)data, DeathRecords);
 293                        break;
 294                    }
 295
 296                case EntityType.PayerPlanPeriod:
 297                    {
 298                        AddEntity((PayerPlanPeriod)data, PayerPlanPeriodsRaw);
 299                        break;
 300                    }
 301
 302                case EntityType.ConditionOccurrence:
 303                    {
 304                        AddEntity((ConditionOccurrence)data, ConditionOccurrencesRaw);
 305                        break;
 306                    }
 307
 308                case EntityType.DrugExposure:
 309                    {
 310                        AddEntity((DrugExposure)data, DrugExposuresRaw);
 311                        break;
 312                    }
 313
 314                case EntityType.ProcedureOccurrence:
 315                    {
 316                        AddEntity((ProcedureOccurrence)data, ProcedureOccurrencesRaw);
 317                        break;
 318                    }
 319
 320                case EntityType.Observation:
 321                    {
 322                        AddEntity((Observation)data, ObservationsRaw);
 323                        break;
 324                    }
 325
 326                case EntityType.VisitOccurrence:
 327                    {
 328                        AddEntity((VisitOccurrence)data, VisitOccurrencesRaw);
 329                        break;
 330                    }
 331
 332                case EntityType.VisitDetail:
 333                    {
 334                        AddEntity((VisitDetail)data, VisitDetailsRaw);
 335                        break;
 336                    }
 337
 338                case EntityType.Cohort:
 339                    {
 340                        AddEntity((Cohort)data, CohortRecords);
 341                        break;
 342                    }
 343
 344                case EntityType.Measurement:
 345                    {
 346                        AddEntity((Measurement)data, MeasurementsRaw);
 347                        break;
 348                    }
 349
 350                case EntityType.DeviceExposure:
 351                    {
 352                        AddEntity((DeviceExposure)data, DeviceExposureRaw);
 353                        break;
 354                    }
 355
 356                case EntityType.Note:
 357                    {
 358                        //AddEntity((Note)data, NoteRecords); TMP: NOTE 
 359                        break;
 360                    }
 361            }
 362        }
 363
 364        protected static void AddEntity<T>(T entity, List<T> list) where T : IEntity
 365        {
 366            list.Add(entity);
 367        }
 368
 369        protected static bool AddEntity<T>(T entity, ConcurrentDictionary<T, T> dictionary) where T : IEntity
 370        {
 371            return dictionary.TryAdd(entity, entity);
 372        }
 373
 374        // set corresponding ProviderIds
 375        protected void SetProviderIds<T>(IEnumerable<T> inputRecords) where T : class, IEntity
 376        {
 377            var records = inputRecords as T[] ?? inputRecords.ToArray();
 378            if (inputRecords == null || !records.Any()) return;
 379
 380
 381            foreach (var e in records.Where(e => !string.IsNullOrEmpty(e.ProviderKey)))
 382            {
 383                e.ProviderId = Entity.GetId(e.ProviderKey);
 384            }
 385        }
 386
 387        // set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
 388        protected virtual void SetPayerPlanPeriodId(PayerPlanPeriod[] payerPlanPeriods, DrugExposure[] drugExposures,
 389            ProcedureOccurrence[] procedureOccurrences, VisitOccurrence[] visitOccurrences,
 390            DeviceExposure[] deviceExposure)
 391        {
 392            if (!payerPlanPeriods.Any()) return;
 393
 394            foreach (var de in drugExposures)
 395            {
 396                if (de.DrugCost == null) continue;
 397                foreach (var planPeriod in payerPlanPeriods)
 398                {
 399                    if (de.StartDate.Between(planPeriod.StartDate, planPeriod.EndDate.Value))
 400                    {
 401                        de.DrugCost.PayerPlanPeriodId = planPeriod.Id;
 402                        break;
 403                    }
 404                }
 405            }
 406
 407            foreach (var po in procedureOccurrences)
 408            {
 409                if (po.ProcedureCosts == null) continue;
 410                foreach (var planPeriod in payerPlanPeriods)
 411                {
 412                    if (po.StartDate.Between(planPeriod.StartDate, planPeriod.EndDate.Value))
 413                    {
 414                        foreach (var procedureCost in po.ProcedureCosts)
 415                        {
 416                            procedureCost.PayerPlanPeriodId = planPeriod.Id;
 417                        }
 418
 419                        break;
 420                    }
 421                }
 422            }
 423
 424            foreach (var vo in visitOccurrences)
 425            {
 426                if (vo.VisitCosts == null) continue;
 427                foreach (var planPeriod in payerPlanPeriods)
 428                {
 429                    if (vo.StartDate.Between(planPeriod.StartDate, planPeriod.EndDate.Value))
 430                    {
 431                        foreach (var visitCost in vo.VisitCosts)
 432                        {
 433                            visitCost.PayerPlanPeriodId = planPeriod.Id;
 434                        }
 435
 436                        break;
 437                    }
 438                }
 439            }
 440
 441            foreach (var de in deviceExposure)
 442            {
 443                if (de.DeviceCosts == null) continue;
 444                foreach (var planPeriod in payerPlanPeriods)
 445                {
 446                    if (de.StartDate.Between(planPeriod.StartDate, planPeriod.EndDate.Value))
 447                    {
 448                        foreach (var deviceCost in de.DeviceCosts)
 449                        {
 450                            deviceCost.PayerPlanPeriodId = planPeriod.Id;
 451                        }
 452
 453                        break;
 454                    }
 455                }
 456            }
 457        }
 458
 459        /// <summary>
 460        /// Projects Enumeration of observation period from the raw set of observation period entities.
 461        /// </summary>
 462        /// <param name="gap">persistence window (duration that is allowed to elapse between two periods) </param>
 463        /// <param name="observationPeriods">raw set of observation period entities</param>
 464        /// <returns>Enumeration of observation period entities</returns>
 465        public virtual IEnumerable<ObservationPeriod> BuildObservationPeriods(int gap, EraEntity[] observationPeriods)
 466        {
 467            if (observationPeriods.Length > 0)
 468            {
 469                yield return new ObservationPeriod
 470                {
 471                    Id = observationPeriods[0].Id,
 472                    PersonId = observationPeriods[0].PersonId,
 473                    StartDate = observationPeriods[0].StartDate,
 474                    EndDate = observationPeriods[0].EndDate.Value,
 475                    TypeConceptId = observationPeriods[0].TypeConceptId
 476                };
 477            }
 478        }
 479
 480        /// <summary>
 481        /// Projects Enumeration of payerPlanPeriod from the raw set of payerPlanPeriod entities.
 482        /// </summary>
 483        /// <param name="payerPlanPeriods">raw set of payerPlanPeriod entities</param>
 484        /// <param name="visitOccurrences">the visit occurrence entities for current person</param>
 485        /// <returns>Enumeration of payerPlanPeriod entities</returns>
 486        public virtual IEnumerable<PayerPlanPeriod> BuildPayerPlanPeriods(PayerPlanPeriod[] payerPlanPeriods,
 487            Dictionary<long, VisitOccurrence> visitOccurrences)
 488        {
 489            // All overlapping periods will be collapsed into one observation period
 490            return EraHelper.GetPayerPlanPeriods(payerPlanPeriods, CanPayerPlanPeriodBeCombined,
 491                Offset);
 492        }
 493
 494        /// <summary>
 495        /// Condition for the combining two periods
 496        /// </summary>
 497        /// <param name="current">1st period</param>
 498        /// <param name="other">2nd period</param>
 499        /// <returns>Can those periods be combined</returns>
 500        public virtual bool CanPayerPlanPeriodBeCombined(PayerPlanPeriod current, PayerPlanPeriod other)
 501        {
 502            if (string.IsNullOrEmpty(current.PlanSourceValue))
 503                return current.PayerSourceValue == other.PayerSourceValue;
 504
 505            return current.PlanSourceValue == other.PlanSourceValue &&
 506                   current.PayerSourceValue == other.PayerSourceValue;
 507        }
 508
 509        /// <summary>
 510        /// Projects death entity from the raw set of death entities.
 511        /// During build:
 512        /// override the death's start date using the end date of the corresponding visit.
 513        /// </summary>
 514        /// <param name="death">raw set of death entities</param>
 515        /// <param name="visitOccurrences">the visit occurrence entities for current person</param>
 516        /// <param name="observationPeriods">the observation period entities for current person</param>
 517        /// <returns>death entity</returns>
 518        public virtual Death BuildDeath(Death[] death, Dictionary<long, VisitOccurrence> visitOccurrences,
 519            ObservationPeriod[] observationPeriods)
 520        {
 521            var ds = Clean(death, observationPeriods, false).ToList();
 522            if (ds.Any())
 523            {
 524                var pd = ds.Where(d => d.Primary).ToList();
 525                return pd.Any() ? pd.OrderBy(d => d.StartDate).Last() : ds.OrderBy(d => d.StartDate).Last();
 526            }
 527
 528            return null;
 529        }
 530
 531        /// <summary>
 532        /// Projects person etity from the raw set of persons entities. 
 533        /// </summary>
 534        /// <param name="records">raw set of Person entities</param>
 535        /// <returns>Person entity</returns>
 536        public virtual KeyValuePair<Person, Attrition> BuildPerson(List<Person> records)
 537        {
 538            if (records == null || records.Count == 0)
 539                return new KeyValuePair<Person, Attrition>(null, Attrition.UnacceptablePatientQuality);
 540
 541            var ordered = records.OrderByDescending(p => p.StartDate).ToArray();
 542            var person = ordered.Take(1).First();
 543            person.StartDate = ordered.Take(1).Last().StartDate;
 544
 545            var gender =
 546                records.GroupBy(p => p.GenderConceptId).OrderByDescending(gp => gp.Count()).Take(1).First().First();
 547            var race = records.GroupBy(p => p.RaceConceptId).OrderByDescending(gp => gp.Count()).Take(1).First()
 548                .First();
 549
 550            person.GenderConceptId = gender.GenderConceptId;
 551            person.GenderSourceValue = gender.GenderSourceValue;
 552            person.RaceConceptId = race.RaceConceptId;
 553            person.RaceSourceValue = race.RaceSourceValue;
 554
 555            if (person.GenderConceptId == 8551) //UNKNOWN
 556            {
 557                return new KeyValuePair<Person, Attrition>(null, Attrition.UnknownGender);
 558            }
 559
 560            return new KeyValuePair<Person, Attrition>(person, Attrition.None);
 561        }
 562
 563        private void AddRawVisitOccurrence(VisitOccurrence rawVisit, VisitOccurrence finalVisit)
 564        {
 565            if (!_rawVisits.ContainsKey(rawVisit.Id))
 566                _rawVisits.Add(rawVisit.Id, finalVisit);
 567            else
 568                _rawVisits[rawVisit.Id] = finalVisit;
 569        }
 570
 571
 572        public IEnumerable<VisitOccurrence> BuildVisitOccurrences(VisitOccurrence[] rawVisitOccurrences,
 573            ObservationPeriod[] observationPeriods)
 574        {
 575            var ipVisits = CollapseVisits(rawVisitOccurrences.Where(vo => vo.ConceptId == 9201), 1).ToList();
 576
 577            var erVisits = new List<VisitOccurrence>();
 578            var opVisits = new List<VisitOccurrence>();
 579            foreach (
 580                var visitOccurrence in
 581                rawVisitOccurrences.Where(
 582                    visitOccurrence => visitOccurrence.ConceptId != 9201))
 583            {
 584                var ip = ipVisits.FirstOrDefault(v => visitOccurrence.StartDate.Between(v.StartDate, v.EndDate.Value));
 585
 586                if (visitOccurrence.ConceptId == 9203)
 587                {
 588                    if (ip == null || (visitOccurrence.StartDate == ip.StartDate &&
 589                                       visitOccurrence.EndDate == ip.StartDate))
 590                    //ER - 9203
 591                    {
 592                        erVisits.Add(visitOccurrence);
 593                    }
 594                    else
 595                    {
 596                        AddRawVisitOccurrence(visitOccurrence, ip);
 597                    }
 598                }
 599                else if (ip == null)
 600                {
 601                    opVisits.Add(visitOccurrence);
 602                }
 603                else
 604                {
 605                    AddRawVisitOccurrence(visitOccurrence, ip);
 606                }
 607            }
 608
 609            foreach (var ipVisit in ipVisits)
 610            {
 611                yield return ipVisit;
 612            }
 613
 614            foreach (var erGroup in erVisits.GroupBy(v => v.StartDate))
 615            {
 616                var visit = erGroup.First();
 617                visit.EndDate = erGroup.Max(v => v.EndDate);
 618                foreach (var visitOccurrence in erGroup)
 619                {
 620                    AddRawVisitOccurrence(visitOccurrence, visit);
 621                }
 622
 623                yield return visit;
 624            }
 625
 626            foreach (var opGroup in opVisits.GroupBy(v => v.StartDate))
 627            {
 628                foreach (var opGroup1 in opGroup.GroupBy(v => v.ProviderKey))
 629                {
 630                    var visit = opGroup1.First();
 631                    visit.EndDate = opGroup1.Max(v => v.EndDate);
 632                    foreach (var visitOccurrence in opGroup1)
 633                    {
 634                        AddRawVisitOccurrence(visitOccurrence, visit);
 635                    }
 636
 637                    yield return visit;
 638                }
 639            }
 640        }
 641
 642        private IEnumerable<VisitOccurrence> CollapseVisits(IEnumerable<VisitOccurrence> visitOccurrences, int gap)
 643        {
 644            var visits = new List<VisitOccurrence>();
 645
 646            foreach (var claim in visitOccurrences.OrderBy(vo => vo.StartDate).ThenBy(vo => vo.EndDate))
 647            {
 648                if (visits.Count > 0)
 649                {
 650                    var previousClaim = visits.Last();
 651                    if (claim.StartDate <= previousClaim.EndDate.Value.AddDays(gap))
 652                    {
 653                        if (claim.EndDate >= previousClaim.EndDate)
 654                        {
 655                            previousClaim.EndDate = claim.EndDate;
 656                        }
 657
 658                        AddRawVisitOccurrence(claim, previousClaim);
 659                        continue;
 660                    }
 661                }
 662
 663                AddRawVisitOccurrence(claim, claim);
 664
 665                visits.Add(claim);
 666            }
 667            return visits;
 668        }
 669
 670        private VisitOccurrence GetVisitOccurrence(IEntity ent)
 671        {
 672            if (ent.VisitOccurrenceId.HasValue && _rawVisits.ContainsKey(ent.VisitOccurrenceId.Value))
 673            {
 674                return _rawVisits[ent.VisitOccurrenceId.Value];
 675            }
 676
 677            return null;
 678        }
 679
 680        public IEnumerable<VisitDetail> BuildVisitDetails(VisitDetail[] visitDetails,
 681         VisitOccurrence[] visitOccurrences, ObservationPeriod[] observationPeriods)
 682        {
 683            foreach (var visitOccurrence in visitOccurrences)
 684            {
 685                var visitDetail =
 686                    new VisitDetail(visitOccurrence)
 687                    {
 688                        Id = visitOccurrence.Id,
 689                        VisitOccurrenceId = visitOccurrence.Id
 690                    };
 691
 692                if (!visitDetail.EndDate.HasValue)
 693                    visitDetail.EndDate = visitDetail.StartDate;
 694
 695
 696                yield return visitDetail;
 697            }
 698        }
 699
 700        public IEnumerable<Note> BuildNote(Note[] notes, Dictionary<long, VisitOccurrence> visitOccurrences,
 701            ObservationPeriod[] observationPeriods)
 702        {
 703            foreach (var e in notes.Where(item =>
 704                observationPeriods.FirstOrDefault(p => item.StartDate.Between(p.StartDate, p.EndDate.Value)) !=
 705                null))
 706            {
 707                if (e.VisitOccurrenceId == null || visitOccurrences.ContainsKey(e.VisitOccurrenceId.Value))
 708                {
 709                    yield return e;
 710                }
 711            }
 712
 713            //return BuildEntities(notes, visitOccurrences, observationPeriods, true); TMP: NOTE 
 714        }
 715
 716        /// <summary>
 717        /// 	Projects Enumeration of drug exposure from the raw set of drug exposure entities. 
 718        /// </summary>
 719        /// <param name="drugExposures">raw set of drug exposures entities</param>
 720        /// <param name="visitOccurrences">the visit occurrences entities for current person</param>
 721        /// <param name="observationPeriods">the observation periods entities for current person</param>
 722        /// <returns>Enumeration of drug exposure entities</returns>
 723        public virtual IEnumerable<DrugExposure> BuildDrugExposures(DrugExposure[] drugExposures,
 724            Dictionary<long, VisitOccurrence> visitOccurrences, ObservationPeriod[] observationPeriods)
 725        {
 726            return BuildEntities(drugExposures, visitOccurrences, observationPeriods, false);
 727        }
 728
 729        /// <summary>
 730        /// Projects Enumeration of ConditionOccurrence from the raw set of ConditionOccurrence entities. 
 731        /// </summary>
 732        /// <param name="conditionOccurrences">raw set of condition occurrence entities</param>
 733        /// <param name="visitOccurrences">the visit occurrence entities for current person</param>
 734        /// <param name="observationPeriods">the observation period entities for current person</param>
 735        /// <returns>Enumeration of condition occurrence entities</returns>
 736        public virtual IEnumerable<ConditionOccurrence> BuildConditionOccurrences(
 737            ConditionOccurrence[] conditionOccurrences, Dictionary<long, VisitOccurrence> visitOccurrences,
 738            ObservationPeriod[] observationPeriods)
 739        {
 740            return BuildEntities(conditionOccurrences, visitOccurrences, observationPeriods, false);
 741        }
 742
 743        /// <summary>
 744        /// Projects Enumeration of ProcedureOccurrence from the raw set of ProcedureOccurence entities.
 745        /// </summary>
 746        /// <param name="procedureOccurrences">raw set of procedure occurrence entities</param>
 747        /// <param name="visitOccurrences">the visit occurrence entities for current person</param>
 748        /// <param name="observationPeriods">the observation period entities for current person</param>
 749        /// <returns>Enumeration of procedure occurrence entities</returns>
 750        public virtual IEnumerable<ProcedureOccurrence> BuildProcedureOccurrences(
 751            ProcedureOccurrence[] procedureOccurrences, Dictionary<long, VisitOccurrence> visitOccurrences,
 752            ObservationPeriod[] observationPeriods)
 753        {
 754            return BuildEntities(procedureOccurrences, visitOccurrences, observationPeriods, false);
 755        }
 756
 757        /// <summary>
 758        /// Projects Enumeration of Observations from the raw set of Observation entities. 
 759        /// </summary>
 760        /// <param name="observations">raw set of observations entities</param>
 761        /// <param name="visitOccurrences">the visit occurrences entities for current person</param>
 762        /// <param name="observationPeriods">the observation periods entities for current person</param>
 763        /// <returns>Enumeration of Observation from the raw set of Observation entities</returns>
 764        public virtual IEnumerable<Observation> BuildObservations(Observation[] observations,
 765            Dictionary<long, VisitOccurrence> visitOccurrences, ObservationPeriod[] observationPeriods)
 766        {
 767            return BuildEntities(observations, visitOccurrences, observationPeriods, false);
 768        }
 769
 770        public virtual IEnumerable<Measurement> BuildMeasurement(Measurement[] measurements,
 771            Dictionary<long, VisitOccurrence> visitOccurrences,
 772            ObservationPeriod[] observationPeriods)
 773        {
 774            return BuildEntities(measurements, visitOccurrences, observationPeriods, false);
 775        }
 776
 777        /// <summary>
 778        /// CONDITION_ERAs are chronological periods of condition occurrence.  
 779        /// There will only be one type of persistence window (duration that is allowed to elapse between condition occurrences) applied to this CDM, which is 30 days. 
 780        /// CONDITION_END_DATE will be the CONDITION_START_DATE.
 781        /// </summary>
 782        /// <param name="conditionOccurrences">Set of condition occurrence entities</param>
 783        /// <param name="observationPeriods">the observation periods entities for current person</param>
 784        /// <returns>Enumeration of condition era</returns>
 785        public virtual IEnumerable<EraEntity> BuildConditionEra(ConditionOccurrence[] conditionOccurrences, ObservationPeriod[] observationPeriods)
 786        {
 787            foreach (var eraEntity in EraHelper.GetEras(
 788                Clean(conditionOccurrences, observationPeriods, false).Where(c => string.IsNullOrEmpty(c.Domain) || c.Domain == "Condition"), 30,
 789                38000247))
 790            {
 791                eraEntity.Id = Offset.GetKeyOffset(eraEntity.PersonId).ConditionEraId;
 792                yield return eraEntity;
 793            }
 794        }
 795
 796        /// <summary>
 797        /// A Drug Era is defined as a span of time when the Person is assumed to be exposed to a particular drug. 
 798        /// Successive periods of Drug Exposures are combined under certain rules to produce continuous Drug Eras.  
 799        /// The Drug Era  is populated by pulling from the set of drug exposure.  A drug era is therefore understood as exposure to a certain compound over a certain period of time. 
 800        /// There will only be one type of persistence window (duration that is allowed to elapse between drug exposures) applied to this CDM, which is 30 days. 
 801        /// </summary>
 802        /// <param name="drugExposures">set of drug exposure entities</param>
 803        /// <param name="observationPeriods">the observation periods entities for current person</param>
 804        /// <returns>Enumeration of drug era</returns>
 805        public virtual IEnumerable<EraEntity> BuildDrugEra(DrugExposure[] drugExposures, ObservationPeriod[] observationPeriods)
 806        {
 807            foreach (
 808                var eraEntity in EraHelper.GetEras(
 809                Clean(drugExposures, observationPeriods, false).Where(d => string.IsNullOrEmpty(d.Domain) || d.Domain == "Drug"), 30, 38000182))
 810            {
 811                eraEntity.Id = Offset.GetKeyOffset(eraEntity.PersonId).DrugEraId;
 812                yield return eraEntity;
 813            }
 814        }
 815
 816        /// <summary>
 817        /// Projects Enumeration of Cohort from the raw set of Cohort entities. 
 818        ///  </summary>
 819        /// <param name="cohort">raw set of Cohort entities</param>
 820        /// <param name="observationPeriods">the observation periods entities for current person</param>
 821        /// <returns>Enumeration of Cohort from the raw set of Cohort entities</returns>
 822        public virtual IEnumerable<Cohort> BuildCohort(Cohort[] cohort, ObservationPeriod[] observationPeriods)
 823        {
 824            return cohort;
 825        }
 826
 827        public IEnumerable<EraEntity> BuildPregnancyEpisodes(ConditionOccurrence[] conditionOccurrences,
 828            ProcedureOccurrence[] procedureOccurrences, Observation[] observations, Measurement[] measurements, DrugExposure[] drugExposures)
 829        {
 830
 831            return null;
 832        }
 833
 834        public virtual IEnumerable<DeviceExposure> BuildDeviceExposure(DeviceExposure[] devExposure,
 835            Dictionary<long, VisitOccurrence> visitOccurrences,
 836            ObservationPeriod[] observationPeriods)
 837        {
 838            return BuildEntities(devExposure, visitOccurrences, observationPeriods, false);
 839        }
 840
 841        /// <summary>
 842        /// Projects Enumeration of drug cost from the set of drug exposure entities.
 843        /// </summary>
 844        /// <param name="drugExposures">set of drug exposure entities</param>
 845        /// <returns>set of drug cost entities</returns>
 846        public virtual IEnumerable<DrugCost> BuildDrugCosts(DrugExposure[] drugExposures)
 847        {
 848            foreach (var drugExposure in drugExposures.Where(drugExposure => drugExposure.DrugCost != null))
 849            {
 850                drugExposure.DrugCost.Id = drugExposure.Id;
 851                yield return drugExposure.DrugCost;
 852            }
 853        }
 854
 855        /// <summary>
 856        /// Projects Enumeration of procedure cost from the set of procedure occurrence entities.
 857        /// </summary>
 858        /// <param name="procedureOccurrences">set of procedure occurrence entities</param>
 859        /// <returns>set of procedure cost entities</returns>
 860        public virtual IEnumerable<ProcedureCost> BuildProcedureCosts(ProcedureOccurrence[] procedureOccurrences)
 861        {
 862            foreach (var po in procedureOccurrences.Where(i => i.ProcedureCosts != null))
 863            {
 864                foreach (var pc in po.ProcedureCosts)
 865                {
 866                    pc.Id = po.Id;
 867                    yield return pc;
 868                }
 869            }
 870        }
 871
 872        public virtual IEnumerable<VisitCost> BuildVisitCosts(VisitOccurrence[] visitOccurrences)
 873        {
 874            foreach (var vo in visitOccurrences.Where(i => i.VisitCosts != null))
 875            {
 876                foreach (var vc in vo.VisitCosts)
 877                {
 878                    vc.Id = vo.Id;
 879                    yield return vc;
 880                }
 881            }
 882        }
 883
 884        public virtual IEnumerable<DeviceCost> BuildDeviceCosts(DeviceExposure[] deviceExposure)
 885        {
 886            foreach (var de in deviceExposure.Where(i => i.DeviceCosts != null))
 887            {
 888                foreach (var dc in de.DeviceCosts)
 889                {
 890                    dc.Id = de.Id;
 891                    yield return dc;
 892                }
 893            }
 894        }
 895
 896        /// <summary>
 897        /// Filtering raw set of entities (DrugExposures, ConditionOccurrences, ProcedureOccurrences...)
 898        /// </summary>
 899        /// <param name="entitiesToBuild">the raw set of entities</param>
 900        /// <param name="visitOccurrences">the visit occurrence entities for current person</param>
 901        /// <param name="observationPeriods">the observation period entities for current person</param>
 902        /// <param name="withinTheObservationPeriod">allow records that are only inside of the observation period</param>
 903        /// <returns>Enumeration of filtered entities</returns>
 904        public virtual IEnumerable<T> BuildEntities<T>(IEnumerable<T> entitiesToBuild,
 905            IDictionary<long, VisitOccurrence> visitOccurrences, IEnumerable<ObservationPeriod> observationPeriods, bool withinTheObservationPeriod)
 906            where T : IEntity
 907        {
 908            var uniqueEntities = new HashSet<T>();
 909            foreach (var e in Clean(entitiesToBuild, observationPeriods, withinTheObservationPeriod))
 910            {
 911                if (e.VisitOccurrenceId == null || visitOccurrences.ContainsKey(e.VisitOccurrenceId.Value))
 912                {
 913                    if (e.IsUnique)
 914                    {
 915                        uniqueEntities.Add(e);
 916                    }
 917                    else
 918                    {
 919                        yield return e;
 920                    }
 921                }
 922            }
 923
 924            foreach (var ue in uniqueEntities)
 925            {
 926                yield return ue;
 927            }
 928        }
 929
 930        /// <summary>
 931        /// Build person entity and all person related entities like: DrugExposures, ConditionOccurrences, ProcedureOccurrences... from raw data sets 
 932        /// </summary>
 933        public virtual Attrition Build(ChunkData data, KeyMasterOffsetManager o)
 934        {
 935            this.Offset = o;
 936            this.ChunkData = data;
 937
 938            var result = BuildPerson(PersonRecords.ToList());
 939            var person = result.Key;
 940            if (person == null)
 941            {
 942                Complete = true;
 943                return result.Value;
 944            }
 945
 946            var observationPeriods =
 947                BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
 948
 949            var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
 950
 951            var visitDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
 952
 953            var visitOccurrences = new Dictionary<long, VisitOccurrence>();
 954            var visitIds = new List<long>();
 955            foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
 956            {
 957                if (!visitOccurrences.ContainsKey(visitOccurrence.Id))
 958                {
 959                    visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
 960                    visitIds.Add(visitOccurrence.Id);
 961                }
 962            }
 963
 964            foreach (var visitDetail in visitDetails)
 965            {
 966                var vo = GetVisitOccurrence(visitDetail);
 967                visitDetail.VisitOccurrenceId = vo?.Id ?? 0;
 968
 969                if (visitDetail.VisitOccurrenceId.HasValue && !_visitDetails.ContainsKey(visitDetail.VisitOccurrenceId.Value))
 970                {
 971                    _visitDetails.Add(visitDetail.VisitOccurrenceId.Value, new List<VisitDetail>());
 972                }
 973
 974                _visitDetails[visitDetail.VisitOccurrenceId.Value].Add(visitDetail);
 975            }
 976
 977            long? prevVisitId = null;
 978            foreach (var visitId in visitIds.OrderBy(v => v))
 979            {
 980                if (prevVisitId.HasValue)
 981                {
 982                    visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
 983                }
 984
 985                prevVisitId = visitId;
 986            }
 987
 988            var drugExposures = new List<DrugExposure>();
 989            foreach (var item in BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods))
 990            {
 991                item.Id = Offset.GetKeyOffset(item.PersonId).DrugExposureId;
 992                drugExposures.Add(item);
 993            }
 994
 995            var conditionOccurrences = new List<ConditionOccurrence>();
 996            foreach (var item in BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods))
 997            {
 998                item.Id = Offset.GetKeyOffset(item.PersonId).ConditionOccurrenceId;
 999                conditionOccurrences.Add(item);
1000            }
1001
1002            var procedureOccurrences = new List<ProcedureOccurrence>();
1003            foreach (var item in BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods))
1004            {
1005                item.Id = Offset.GetKeyOffset(item.PersonId).ProcedureOccurrenceId;
1006                procedureOccurrences.Add(item);
1007            }
1008
1009            var observations = new List<Observation>();
1010            foreach (var item in BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods))
1011            {
1012                item.Id = Offset.GetKeyOffset(item.PersonId).ObservationId;
1013                observations.Add(item);
1014            }
1015
1016            var measurements = new List<Measurement>();
1017            foreach (var item in BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods))
1018            {
1019                item.Id = Offset.GetKeyOffset(item.PersonId).MeasurementId;
1020                measurements.Add(item);
1021            }
1022
1023            var deviceExposure =
1024                BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
1025
1026            // set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
1027            SetPayerPlanPeriodId(payerPlanPeriods, drugExposures.ToArray(), procedureOccurrences.ToArray(),
1028                visitOccurrences.Values.ToArray(),
1029                deviceExposure);
1030
1031            // set corresponding ProviderIds
1032            SetProviderIds(drugExposures);
1033            SetProviderIds(conditionOccurrences);
1034            SetProviderIds(procedureOccurrences);
1035            SetProviderIds(observations);
1036
1037            var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
1038
1039            var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
1040            var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
1041
1042            List<DateTime?> mins = new List<DateTime?>();
1043            List<DateTime?> maxs = new List<DateTime?>();
1044
1045            mins.Add(GetMinDate(drugExposures));
1046            mins.Add(GetMinDate(conditionOccurrences));
1047            mins.Add(GetMinDate(procedureOccurrences));
1048            mins.Add(GetMinDate(observations));
1049            mins.Add(GetMinDate(deviceExposure));
1050            mins.Add(GetMinDate(measurements));
1051            mins.Add(GetMinDate(visitOccurrences.Values));
1052            mins.Add(GetMinDate(visitDetails));
1053
1054            maxs.Add(GetMaxDate(drugExposures));
1055            maxs.Add(GetMaxDate(conditionOccurrences));
1056            maxs.Add(GetMaxDate(procedureOccurrences));
1057            maxs.Add(GetMaxDate(observations));
1058            maxs.Add(GetMaxDate(deviceExposure));
1059            maxs.Add(GetMaxDate(measurements));
1060            maxs.Add(GetMaxDate(visitOccurrences.Values));
1061            maxs.Add(GetMaxDate(visitDetails));
1062
1063            var min = mins.Min();
1064            var max = maxs.Max();
1065
1066            var observationPeriodsFinal = new List<ObservationPeriod>(1)
1067            {
1068                new ObservationPeriod {PersonId = person.PersonId, TypeConceptId = 32817 }
1069            };
1070
1071            if (min.HasValue)
1072            {
1073                observationPeriodsFinal[0].StartDate = min.Value;
1074
1075                if (max.HasValue)
1076                    observationPeriodsFinal[0].EndDate = max;
1077                else
1078                    observationPeriodsFinal[0].EndDate = min.Value;
1079            }
1080
1081            SetVisitOccurrenceId(drugExposures);
1082            SetVisitOccurrenceId(conditionOccurrences);
1083            SetVisitOccurrenceId(procedureOccurrences);
1084            SetVisitOccurrenceId(measurements);
1085            SetVisitOccurrenceId(observations);
1086            SetVisitOccurrenceId(deviceExposure);
1087
1088            // push built entities to ChunkBuilder for further save to CDM database
1089            AddToChunk(person, death, observationPeriodsFinal.ToArray(), payerPlanPeriods, drugExposures.ToArray(),
1090                conditionOccurrences.ToArray(), procedureOccurrences.ToArray(), observations.ToArray(), measurements.ToArray(),
1091                visitOccurrences.Values.ToArray(), visitDetails, cohort, deviceExposure, notes);
1092
1093            Complete = true;
1094
1095            return Attrition.None;
1096        }
1097
1098        private void SetVisitOccurrenceId(IEnumerable<IEntity> entities)
1099        {
1100            Parallel.ForEach(entities, e =>
1101            {
1102                var vo = GetVisitOccurrence(e);
1103
1104                if (vo != null)
1105                {
1106                    e.VisitOccurrenceId = vo.Id;
1107                    e.ProviderId = vo.ProviderId;
1108                }
1109                else
1110                {
1111                    e.VisitOccurrenceId = null;
1112                }
1113            });
1114        }
1115
1116        protected DateTime? GetMinDate<T>(IEnumerable<T> inputRecords) where T : class, IEntity
1117        {
1118            if (inputRecords == null || inputRecords.Count() == 0)
1119                return null;
1120
1121            return inputRecords.Min(e => e.StartDate);
1122        }
1123
1124        protected DateTime? GetMaxDate<T>(IEnumerable<T> inputRecords) where T : class, IEntity
1125        {
1126            if (inputRecords == null || inputRecords.Count() == 0)
1127                return null;
1128
1129            if (inputRecords.Any(e => e.EndDate.HasValue))
1130                return inputRecords.Where(e => e.EndDate.HasValue).Max(e => e.EndDate.Value);
1131
1132            return null;
1133        }
1134
1135        protected void AddToChunk(Person person, Death death, ObservationPeriod[] observationPeriods,
1136            PayerPlanPeriod[] ppp, DrugExposure[] drugExposures,
1137            ConditionOccurrence[] conditionOccurrences,
1138            ProcedureOccurrence[] procedureOccurrences, Observation[] observations,
1139            VisitOccurrence[] visitOccurrences, Cohort[] cohort)
1140        {
1141            AddToChunk(person, death, observationPeriods,
1142                ppp, drugExposures,
1143                conditionOccurrences,
1144                procedureOccurrences, observations, new Measurement[] { },
1145                visitOccurrences, null, cohort, new DeviceExposure[] { },
1146                new Note[] { });
1147        }
1148
1149        /// <summary>
1150        /// Push built entities to ChunkBuilder for further save to CDM database
1151        /// </summary>
1152        /// <param name="person">person entity</param>
1153        /// <param name="death">death entity</param>
1154        /// <param name="observationPeriods">the observation period entities for current person</param>
1155        /// <param name="ppp">the payerplan period entities for current person</param>
1156        /// <param name="drugExposures">the drug exposure entities for current person</param>
1157        /// <param name="conditionOccurrences">the condition occurrence entities for current person</param>
1158        /// <param name="procedureOccurrences">the procedure occurrence entities for current person</param>
1159        /// <param name="observations">the observation entities for current person</param>
1160        /// <param name="visitOccurrences">the visit occurrence entities for current person</param>
1161
1162        /// <param name="cohort">the cohort entities for current person</param>
1163        public virtual void AddToChunk(Person person, Death death, ObservationPeriod[] observationPeriods,
1164            PayerPlanPeriod[] ppp, DrugExposure[] drugExposures,
1165            ConditionOccurrence[] conditionOccurrences,
1166            ProcedureOccurrence[] procedureOccurrences, Observation[] observations,
1167            Measurement[] measurements, VisitOccurrence[] visitOccurrences, VisitDetail[] visitDetails, Cohort[] cohort,
1168            DeviceExposure[] devExposure, Note[] notes)
1169        {
1170            ChunkData.AddData(person);
1171
1172            if (death != null)
1173            {
1174                if (!death.CauseConceptId.HasValue)
1175                    death.CauseConceptId = 0;
1176
1177                ChunkData.AddData(death);
1178            }
1179
1180            foreach (var observationPeriod in observationPeriods)
1181            {
1182                ChunkData.AddData(observationPeriod);
1183            }
1184
1185            foreach (var payerPlanPeriod in ppp)
1186            {
1187                ChunkData.AddData(payerPlanPeriod);
1188            }
1189
1190            foreach (var visitOccurrence in visitOccurrences)
1191            {
1192                ChunkData.AddData(visitOccurrence);
1193            }
1194
1195            if (visitDetails != null)
1196            {
1197                foreach (var visitDetail in visitDetails)
1198                {
1199                    ChunkData.AddData(visitDetail);
1200                }
1201            }
1202
1203            if (cohort != null)
1204            {
1205                foreach (var c in cohort)
1206                {
1207                    ChunkData.AddData(c);
1208                }
1209            }
1210
1211            if (notes != null)
1212            {
1213                foreach (var n in notes)
1214                {
1215                    ChunkData.Note.Add(n);
1216                    //TMP: NOTE
1217                    //ChunkData.AddData(n);
1218                }
1219            }
1220
1221            AddToChunk("Condition", conditionOccurrences);
1222            AddToChunk("Drug", drugExposures);
1223            AddToChunk("Procedure", procedureOccurrences);
1224            AddToChunk("Observation", observations);
1225            AddToChunk("Measurement", measurements);
1226            AddToChunk("Device", devExposure);
1227
1228            var drugEra = BuildDrugEra(DrugForEra.ToArray(), observationPeriods).ToArray();
1229            var conditionEra = BuildConditionEra(ConditionForEra.ToArray(), observationPeriods).ToArray();
1230
1231            foreach (var eraEntity in drugEra)
1232            {
1233                ChunkData.AddData(eraEntity, EntityType.DrugEra);
1234            }
1235
1236            foreach (var eraEntity in conditionEra)
1237            {
1238                ChunkData.AddData(eraEntity, EntityType.ConditionEra);
1239            }
1240        }
1241
1242        public string GetDomain(string entityDomain, string conceptDomain)
1243        {
1244            switch (conceptDomain)
1245            {
1246                case "Condition":
1247                case "Measurement":
1248                case "Meas Value":
1249                case "Observation":
1250                case "Procedure":
1251                case "Device":
1252                case "Drug":
1253                    return conceptDomain;
1254
1255                default:
1256                    return entityDomain;
1257            }
1258        }
1259
1260        public virtual void AddToChunk(string domain, IEnumerable<IEntity> entities)
1261        {
1262            foreach (var entity in entities)
1263            {
1264                var entityDomain = GetDomain(domain, entity.Domain);
1265
1266                switch (entityDomain)
1267                {
1268                    case "Condition":
1269                        var obs = entity as Observation;
1270                        if (obs == null || obs.ValueAsNumber == 1)
1271                        {
1272                            var cond = entity as ConditionOccurrence ??
1273                                       new ConditionOccurrence(entity)
1274                                       {
1275                                           Id = Offset.GetKeyOffset(entity.PersonId).ConditionOccurrenceId
1276                                       };
1277                            ConditionForEra.Add(cond);
1278                            ChunkData.AddData(cond);
1279                        }
1280
1281                        break;
1282
1283                    case "Measurement":
1284                        ChunkData.AddData(entity as Measurement ?? new Measurement(entity)
1285                        {
1286                            Id = Offset.GetKeyOffset(entity.PersonId).MeasurementId
1287                        });
1288                        break;
1289
1290                    case "Meas Value":
1291                        ChunkData.AddData(entity as Measurement ?? new Measurement(entity)
1292                        {
1293                            Id = Offset.GetKeyOffset(entity.PersonId).MeasurementId
1294                        });
1295                        break;
1296
1297                    case "Observation":
1298                        ChunkData.AddData(entity as Observation ?? new Observation(entity)
1299                        {
1300                            Id = Offset.GetKeyOffset(entity.PersonId).ObservationId
1301                        });
1302                        break;
1303
1304                    case "Procedure":
1305                        ChunkData.AddData(entity as ProcedureOccurrence ??
1306                                          new ProcedureOccurrence(entity)
1307                                          {
1308                                              Id =
1309                                                  Offset.GetKeyOffset(entity.PersonId).ProcedureOccurrenceId
1310                                          });
1311                        break;
1312
1313                    case "Device":
1314                        ChunkData.AddData(entity as DeviceExposure ??
1315                                          new DeviceExposure(entity)
1316                                          {
1317                                              Id = Offset.GetKeyOffset(entity.PersonId).DeviceExposureId
1318                                          });
1319                        break;
1320
1321                    case "Drug":
1322                        var drg = entity as DrugExposure ??
1323                                  new DrugExposure(entity)
1324                                  {
1325                                      Id = Offset.GetKeyOffset(entity.PersonId).DrugExposureId
1326                                  };
1327
1328                        if (!drg.EndDate.HasValue)
1329                            drg.EndDate = drg.StartDate;
1330
1331                        DrugForEra.Add(drg);
1332                        ChunkData.AddData(drg);
1333                        break;
1334
1335                }
1336
1337            }
1338        }
1339
1340
1341        /// exclude records that out of any available observation period records
1342        protected static IEnumerable<T> Clean<T>(IEnumerable<T> input,
1343            IEnumerable<ObservationPeriod> observationPeriods, bool withinTheObservationPeriod) where T : IEntity
1344        {
1345            if (withinTheObservationPeriod)
1346            {
1347                return input.Where(item =>
1348                    observationPeriods.FirstOrDefault(p => item.StartDate.Between(p.StartDate, p.EndDate.Value)) !=
1349                    null);
1350            }
1351
1352            return input;
1353        }
1354
1355        /// <summary>
1356        /// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
1357        /// </summary>
1358        /// <param name="person">the current person entity</param>
1359        /// <param name="periods">the observation period entities for current person</param>
1360        /// <returns>Enumeration of filtered entities</returns>
1361        //public virtual bool Excluded(Person person, IEnumerable<ObservationPeriod> periods)
1362        //{
1363        //    return periods.Any(period => person.YearOfBirth - period.StartDate.Year >= 2);
1364        //}
1365
1366        public void JoinToVocabulary(IVocabulary vocabulary)
1367        {
1368            if (Vocabulary == null)
1369                Vocabulary = vocabulary;
1370        }
1371
1372        public bool Excluded(Person person, IEnumerable<ObservationPeriod> periods)
1373        {
1374            return false;
1375        }
1376
1377        #endregion
1378    }
1379}
1380