Uploaded image for project: 'Grouper'
  1. Grouper
  2. GRP-4150

Export of loader job won't import

    XMLWordPrintable

Details

    • Bug
    • Resolution: Unresolved
    • Minor
    • None
    • 2.6.9a
    • gsh
    • None
    • 2.6.9 latest containers

    Description

      See https://internet2.slack.com/archives/C7V0UQDJ4/p1657571863131669

      The last comment is:
      Wow - i’m stumped.  will submit Jira.  I ended up removing ALL comments and all newlines - back to one long SQL statement… and it STILL fails on the import.  I am going to submit a jira with the one long line of SQL and see if that helps.

      I'll attach the script which fails.

      :set verbosity quiet
      /* imports for use under Java, or in groovy if evaluating as an external file */
      import edu.internet2.middleware.grouper.*;
      import edu.internet2.middleware.grouper.attr.*;
      import edu.internet2.middleware.grouper.attr.assign.AttributeAssign;
      import edu.internet2.middleware.grouper.attr.assign.AttributeAssignSave;
      import edu.internet2.middleware.grouper.attr.assign.AttributeAssignType;
      import edu.internet2.middleware.grouper.attr.assign.AttributeAssignAction;
      import edu.internet2.middleware.grouper.attr.finder.AttributeDefFinder;
      import edu.internet2.middleware.grouper.attr.finder.AttributeDefNameFinder;
      import edu.internet2.middleware.grouper.group.CompositeSave;
      import edu.internet2.middleware.grouper.group.TypeOfGroup;
      import edu.internet2.middleware.grouper.misc.CompositeType;
      import edu.internet2.middleware.grouper.misc.SaveResultType;
      import edu.internet2.middleware.grouper.privs.Privilege;
      import edu.internet2.middleware.grouper.util.GrouperUtil;
      import edu.internet2.middleware.subject.Subject;
      import java.util.HashSet;
      import java.util.Set;
      import java.util.Date;
       
       
      GrouperSession grouperSession = GrouperSession.startRootSession();
      long gshTotalObjectCount = 0L;
      long gshTotalChangeCount = 0L;
      long gshTotalErrorCount = 0L;
       
       
      StemSave stemSave = null;
      Stem stem = null;
      Stem ownerStem = null;
      GroupSave groupSave = null;
      Group group = null;
      Group ownerGroup = null;
      Group leftFactorGroup = null;
      Group rightFactorGroup = null;
      Group ifHasRole = null;
      Group thenHasRole = null;
      CompositeType compositeType = null;
      AttributeDefSave attributeDefSave = null;
      AttributeDef attributeDef = null;
      AttributeDef ownerAttributeDef = null;
      Privilege privilege = null;
      Subject subject = null;
      Subject ownerSubject = null;
      AttributeDefNameSave attributeDefNameSave = null;
      AttributeDefName attributeDefName = null;
      AttributeDefName ifHasAttributeDefName = null;
      AttributeDefName thenHasAttributeDefName = null;
      AttributeDefScopeType attributeDefScopeType = null;
      AttributeAssignSave attributeAssignSave = null;
      AttributeAssignSave attributeAssignOnAssignSave = null;
      boolean problemWithAttributeAssign = false;
       
       
      System.out.println(new Date().toString() + " Done with folders, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      groupSave = new GroupSave(grouperSession).assignName("etc:loaders:Unix_PosixGroups").assignCreateParentStemsIfNotExist(true).assignDisplayName("etc:loaders:Unix_PosixGroups").assignTypeOfGroup(TypeOfGroup.group);
      group = groupSave.save();
      gshTotalObjectCount++;
      if (groupSave.getSaveResultType() != SaveResultType.NO_CHANGE) { System.out.println("Made change for group: " + group.getName()); gshTotalChangeCount++;}
      System.out.println(new Date().toString() + " Done with groups, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with composites, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with attribute definitions, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with role hierarchies, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with attribute actions, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with attribute action hierarchies, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with memberships and privileges, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with attribute names, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with attribute name hierarchies, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      System.out.println(new Date().toString() + " Done with attribute definition scopes, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount);
      Set<String> attributeAssignIdsAlreadyUsed = new HashSet<>();
      problemWithAttributeAssign = false;
      attributeAssignSave = new AttributeAssignSave(grouperSession).assignAttributeAssignIdsToNotUse(attributeAssignIdsAlreadyUsed).assignPrintChangesToSystemOut(true);
      attributeAssignSave.assignAttributeAssignType(AttributeAssignType.group);
      attributeDefName = AttributeDefNameFinder.findByName("etc:legacy:attribute:legacyGroupType_grouperLoader", false);
      if (attributeDefName == null) { gshTotalErrorCount++;  System.out.println("Error: cant find attributeDefName: etc:legacy:attribute:legacyGroupType_grouperLoader");  problemWithAttributeAssign = true; }
      attributeAssignSave.assignAttributeDefName(attributeDefName);
      ownerGroup = GroupFinder.findByName(grouperSession, "etc:loaders:Unix_PosixGroups", false);
      if (ownerGroup == null) { gshTotalErrorCount++; System.out.println("Error: cant find group: etc:loaders:Unix_PosixGroups"); problemWithAttributeAssign = true;  }
      attributeAssignSave.assignOwnerGroup(ownerGroup);
      attributeAssignSave.assignPutAttributeAssignIdsToNotUseSet(true);
      attributeAssignOnAssignSave = new AttributeAssignSave(grouperSession).assignAttributeAssignIdsToNotUse(attributeAssignIdsAlreadyUsed).assignPrintChangesToSystemOut(true);
      attributeAssignOnAssignSave.assignAttributeAssignType(AttributeAssignType.group_asgn);
      attributeDefName = AttributeDefNameFinder.findByName("etc:legacy:attribute:legacyAttribute_grouperLoaderQuartzCron", false);
      if (attributeDefName == null) { gshTotalErrorCount++;  System.out.println("Error: cant find attributeDefName: etc:legacy:attribute:legacyAttribute_grouperLoaderQuartzCron");  problemWithAttributeAssign = true; }
      attributeAssignOnAssignSave.assignAttributeDefName(attributeDefName);
      attributeAssignOnAssignSave.assignPutAttributeAssignIdsToNotUseSet(true);
      attributeAssignOnAssignSave.addValue("0 0/15 6-20 * * ?");
      attributeAssignSave.addAttributeAssignOnThisAssignment(attributeAssignOnAssignSave);
      attributeAssignOnAssignSave = new AttributeAssignSave(grouperSession).assignAttributeAssignIdsToNotUse(attributeAssignIdsAlreadyUsed).assignPrintChangesToSystemOut(true);
      attributeAssignOnAssignSave.assignAttributeAssignType(AttributeAssignType.group_asgn);
      attributeDefName = AttributeDefNameFinder.findByName("etc:legacy:attribute:legacyAttribute_grouperLoaderQuery", false);
      if (attributeDefName == null) { gshTotalErrorCount++;  System.out.println("Error: cant find attributeDefName: etc:legacy:attribute:legacyAttribute_grouperLoaderQuery");  problemWithAttributeAssign = true; }
      attributeAssignOnAssignSave.assignAttributeDefName(attributeDefName);
      attributeAssignOnAssignSave.assignPutAttributeAssignIdsToNotUseSet(true);
      attributeAssignOnAssignSave.addValue("select distinct group_name, subject_source_id, subject_id , unixgroups.gidnumber from( with gid_dups as ( select gidnumber as gid, count(*) from ldap_unix_groups lug group by gid having count(*) > 1), all_groups as ( select distinct Group_name, subject_id, gidnumber from( select lug.cn as Group_name , b.but_lid || '@slac.stanford.edu' as subject_id , lug.gidnumber from ldap_unix_groups_attrs luga, ldap_unix_groups lug , ldap_unix_accounts lua, but b, iam_registry_enabled_v irev where luga.ldap_id = lug.the_dn and luga.attribute_value = lua.uid and lua.uidnumber = b.but_uid and b.but_ldt = 'unix' and b.but_stat = 'E' and b.but_sid = irev.sid and irev.hrstat = 'ACTIVE' union all /* bring primary groups from but into the list */ select distinct but_grp as Group_name, b.but_lid || '@slac.stanford.edu' as subject_id, lug.gidnumber from but b, iam_registry_enabled_v irev, ldap_unix_groups lug where b.but_grp = lug.cn and irev.sid = b.but_sid and b.but_ldt = 'unix' and b.but_stat = 'E' and irev.hrstat = 'ACTIVE' /* do this once to force the creation of all unix groups (false becomes true) * then change true to false to remove * root@slac from all groups */ union all select lug.cn as Group_name , 'root@slac.stanford.edu' subject_id , lug.gidnumber from ldap_unix_groups lug where false) as allgroups) select 'basis:Unix:posixGroups:' || Group_name as Group_name , 'slacPerson' as subject_source_id, subject_id, gidnumber from all_groups ag where not exists (select 1 from gid_dups gd2 where ag.gidnumber = gd2.gid) union all select 'basis:Unix:posixGroups:' || case when Group_name like 'glpwr-%' then 'glast-power' when Group_name like 'glgr-%' then 'glground' when Group_name like 'bfact-%' then 'bfactory' else regexp_replace(Group_name, '(-[a-z])$', '') end as Group_name , 'slacPerson' as subject_source_id, subject_id, gidnumber from all_groups ag, gid_dups gd where ag.gidnumber = gd.gid union all /* all accounts in any group into 1 */ select distinct 'basis:Unix:ActiveAccountsFromGroups' as Group_name , 'slacPerson' as subject_source_id , subject_id , 0 as gidnumber from all_groups ag) as unixgroups");
      attributeAssignSave.addAttributeAssignOnThisAssignment(attributeAssignOnAssignSave);
      attributeAssignOnAssignSave = new AttributeAssignSave(grouperSession).assignAttributeAssignIdsToNotUse(attributeAssignIdsAlreadyUsed).assignPrintChangesToSystemOut(true);
      attributeAssignOnAssignSave.assignAttributeAssignType(AttributeAssignType.group_asgn);
      attributeDefName = AttributeDefNameFinder.findByName("etc:legacy:attribute:legacyAttribute_grouperLoaderType", false);
      if (attributeDefName == null) { gshTotalErrorCount++;  System.out.println("Error: cant find attributeDefName: etc:legacy:attribute:legacyAttribute_grouperLoaderType");  problemWithAttributeAssign = true; }
      attributeAssignOnAssignSave.assignAttributeDefName(attributeDefName);
      attributeAssignOnAssignSave.assignPutAttributeAssignIdsToNotUseSet(true);
      attributeAssignOnAssignSave.addValue("SQL_GROUP_LIST");
      attributeAssignSave.addAttributeAssignOnThisAssignment(attributeAssignOnAssignSave);
      attributeAssignOnAssignSave = new AttributeAssignSave(grouperSession).assignAttributeAssignIdsToNotUse(attributeAssignIdsAlreadyUsed).assignPrintChangesToSystemOut(true);
      attributeAssignOnAssignSave.assignAttributeAssignType(AttributeAssignType.group_asgn);
      attributeDefName = AttributeDefNameFinder.findByName("etc:legacy:attribute:legacyAttribute_grouperLoaderScheduleType", false);
      if (attributeDefName == null) { gshTotalErrorCount++;  System.out.println("Error: cant find attributeDefName: etc:legacy:attribute:legacyAttribute_grouperLoaderScheduleType");  problemWithAttributeAssign = true; }
      attributeAssignOnAssignSave.assignAttributeDefName(attributeDefName);
      attributeAssignOnAssignSave.assignPutAttributeAssignIdsToNotUseSet(true);
      attributeAssignOnAssignSave.addValue("CRON");
      attributeAssignSave.addAttributeAssignOnThisAssignment(attributeAssignOnAssignSave);
      attributeAssignOnAssignSave = new AttributeAssignSave(grouperSession).assignAttributeAssignIdsToNotUse(attributeAssignIdsAlreadyUsed).assignPrintChangesToSystemOut(true);
      attributeAssignOnAssignSave.assignAttributeAssignType(AttributeAssignType.group_asgn);
      attributeDefName = AttributeDefNameFinder.findByName("etc:legacy:attribute:legacyAttribute_grouperLoaderDbName", false);
      if (attributeDefName == null) { gshTotalErrorCount++;  System.out.println("Error: cant find attributeDefName: etc:legacy:attribute:legacyAttribute_grouperLoaderDbName");  problemWithAttributeAssign = true; }
      attributeAssignOnAssignSave.assignAttributeDefName(attributeDefName);
      attributeAssignOnAssignSave.assignPutAttributeAssignIdsToNotUseSet(true);
      attributeAssignOnAssignSave.addValue("iam");
      attributeAssignSave.addAttributeAssignOnThisAssignment(attributeAssignOnAssignSave);
      gshTotalObjectCount += 11;
      if (!problemWithAttributeAssign) { AttributeAssign attributeAssign = attributeAssignSave.save(); if (attributeAssignSave.getChangesCount() > 0) { gshTotalChangeCount+=attributeAssignSave.getChangesCount();  System.out.println("Made " + attributeAssignSave.getChangesCount() + " changes for attribute assign: " + attributeAssign.toString()); } }
      System.out.println(new Date().toString() + " Script complete: total objects, objects: " + gshTotalObjectCount + ", expected approx total: 12, changes: " + gshTotalChangeCount + ", known errors (view output for full list): " + gshTotalErrorCount); 

      Attachments

        Activity

          People

            chris.hyzer@at.internet2.edu Chris Hyzer (upenn.edu)
            michael.gettes.4@at.internet2.edu Michael Gettes (ufl.edu)
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

            Dates

              Created:
              Updated: