U Lde@sddlZddlZddlZddlZddlZddlZddlmZddlm Z m Z ddl m Z m Z mZmZmZddlmZmZmZddlmZddlmZmZddlmZdd lmZdd lmZdd lmZdd l m!Z!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*dZ+dZ,dZ-dZ.e+e,e-gZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7e8e9Z:dddfd d!dfd"d#dfd$Z;e Gd%d&d&e ZGd)d*d*e=Z?Gd+d,d,e=Z@dYd/d0ZAe4fd1d2ZBed3d4d5d6d7gZCed8d9d:gZDGd;dde(ejEd<ZFd=d>ZGeeFeHfd?d@dAZIdBdCZJdZeHeKdEdFdGZLdHdIZMd[dKdLZNGdMdNdNeOZPdOdPZQeFeHeKdQdRdSZReHeeFdTdUdVZSeHd?dWdXZTdS)\N) namedtuple)Enumunique)AnyDictListOptionalTuple) atomic_helperdmiimporter)log)net type_utils) user_data)util) write_json)Distro) EventScope EventType) launch_index)Paths)CloudInitPickleMixin)eventsZdisabledlocalrpassZ FILESYSTEMNETWORK DataSourcez|EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.zredacted for non-root user cloud-nameZ_unsetunknownz aws-chinacCs|dkSNZawscr!r!@r%zaws-govcCs|dkSr r!r"r!r!r$r%Ar&z azure-chinacCs|dkS)NZazurer!r"r!r!r$r%Br&)zcn-zus-gov-Zchinac@s2eZdZdZdZdZdZdZdZe ddd Z d S) NetworkConfigSourcezb Represents the canonical list of network config sources that cloud-init knows about. cmdlinedsZ system_cfgZfallbackZ initramfsreturncCs|jSNvalueselfr!r!r$__str__SszNetworkConfigSource.__str__N) __name__ __module__ __qualname____doc__CMD_LINEDS SYSTEM_CFGZFALLBACK INITRAMFSstrr1r!r!r!r$r'Fsr'c@seZdZdZdS)DatasourceUnpickleUserDataErrorzERaised when userdata is unable to be unpickled due to python upgradesNr2r3r4r5r!r!r!r$r;Wsr;c@s eZdZdS)DataSourceNotFoundExceptionNr2r3r4r!r!r!r$r=[sr=c@seZdZdZdS)InvalidMetaDataExceptionz8Raised when metadata is broken, unavailable or disabled.Nr<r!r!r!r$r?_sr?r!c Cst|}g}g}|D]\}}|r4|d|}n|}||ksP||krZ||t|tr|dr|||dd||<t|t rt |||} | | d| | d| ||<qt ||d<t ||d<|S)zProcess all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @return Dict copy of processed metadata. /zci-b64:r@base64_encoded_keyssensitive_keys)copydeepcopyitemslowerappend isinstancer: startswithreplacedictprocess_instance_metadataextendpopsorted) metadatakey_pathrCmd_copyrBZ sens_keyskeyvalZ sub_key_pathZ return_valr!r!r$rMcs6         rMcCs|dgs|St|}|dD]V}|d}|}|D].}||kr:t||tr:||dkr:||}q:||kr$|||<q$|S)zRedact any sensitive keys from to provided metadata dictionary. Replace any keys values listed in 'sensitive_keys' with redact_value. rCrA)getrDrEsplitrIrL)rQZ redact_valuerSrR path_partsobjpathr!r!r$redact_sensitive_keyss"       r\ URLParamsZmax_wait_secondsZtimeout_secondsZ num_retriessec_between_retriesDataSourceHostnamehostname is_defaultc @seZdZUeZdZdZdZdZdZ dZ e j e j e je jfZee dfed<dZdZdZd ZejejejejejhiZejejhiZd e fd e fd ifd dddddf Z!eee"e#fdfed<dZ$dZ%ee"dfed<dZ&d Z'dxe(e)dddZ*e+ddddZ,ddZ-e.dd d!Z/e.dd"d#Z0d$d%Z1d&d'Z2dyd)d*Z3e.dd+d,Z4dzd.d/Z5e.dd0d1Z6d2d3Z7d{d4d5Z8d6d7Z9d8d9Z:e;d:d;Zd?Z>d@dAZ?e;dBdCZ@dDdEZAe;dFdGZBdHdIZCe;dJdKZDdLdMZEdNdOZFdPdQZGdRdSZHdTdUZIdVdWZJdXdYZKdZd[ZLd\d]ZMe;d^d_ZNe;d`daZOdbdcZPd|dddeZQdfdgZReSedhdidjZTeSee.dkdldmZUdndoZVeWd}dpdqZXe;drdsZYdtduZZdvdwZ[dS)~rz en_US.UTF-8Z_undefN.network_config_sourcesrV  ec2_metadata network_jsonrQ)userdataN) userdata_rawN) vendordataN)vendordata_rawN) vendordata2N)vendordata2_rawNcached_attr_defaultsF) Zcombined_cloud_config merged_cfgmerged_system_cfgzsecurity-credentialsrh user-datarrj vendor-datazds/vendor_datasensitive_metadata_keys)distropathscCs~||_||_||_d|_i|_d|_d|_d|_d|_d|_ t |jd|j fi|_ |j s`i|_ |stt|j|_n||_dS)NZ datasource)sys_cfgrtrurhrQrirjrlrkrmrZget_cfg_by_pathdsnameds_cfgudZUserDataProcessorud_proc)r0rvrtrurzr!r!r$__init__s(zDataSource.__init__)ci_pkl_versionr+c Cst|dsd|_t|ds d|_t|ds0d|_t|dr|jdk rzt|jWn6tk r}ztd|t |W5d}~XYnXdS)z(Perform deserialization fixes for Paths.rlNrmskip_hotplug_detectFrhz:Unable to unpickle datasource: %s. Ignoring current cache.) hasattrrlrmr}rhr:AttributeErrorLOGdebugr;)r0r|er!r!r$ _unpickle4s   zDataSource._unpicklecCs t|Sr,robj_namer/r!r!r$r1LszDataSource.__str__r*cCsdS)z#Check if running on this datasourceTr!r/r!r!r$ ds_detectOszDataSource.ds_detectcCsX|jtkr$td|dS|jdg|jg|jdgfkrTtd|dSdS)aIOverride if either: - only a single datasource defined (nothing to fall back to) - commandline argument is used (ci.ds=OpenStack) Note: get_cmdline() is required for the general case - when ds-identify does not run, _something_ needs to detect the kernel command line definition. zOMachine is configured by the kernel commandline to run on single datasource %s.TZdatasource_listNonez5Machine is configured to run on single datasource %s.F)rwrG parse_cmdlinerrrvrWr/r!r!r$override_ds_detectSs  zDataSource.override_ds_detectcCs@|r|S|r,td||Std|dSdS)z&Overrides runtime datasource detectionz8Detected platform: %s. Checking for active instance dataz#Datasource type %s is not detected.FN)r _get_datarrrr/r!r!r$_check_and_get_datams zDataSource._check_and_get_datacCs|j}|}|j}|d}ddg||t|j|j|j|j|j|dd|dd|dd|j||d|||d d|||d d |j|j |d |d d iS)z2Return a dictionary of standardized metadata keys.sys_infov1 subplatformZdistrrepythonunameplatformvariant)Z _beta_keysavailability-zoneavailability_zonecloud_idr cloud_namertZdistro_versionZdistro_releaserZpublic_ssh_keysZpython_version instance-id instance_idZkernel_releaselocal-hostnamelocal_hostnamemachineregionrZsystem_platformr) get_hostnamer`get_instance_idrcanonical_cloud_idrr platform_typeget_public_ssh_keysr)r0 instance_datarrrZsysinfor!r!r$_get_standardized_metadata{sB      z%DataSource._get_standardized_metadatar!cCsL|js dS|r|}n|j}|D]\}}t||rt|||q|sHd|_dS)zReset any cached metadata attributes to datasource defaults. @param attr_defaults: Optional tuple of (attr, value) pairs to set instead of cached_attr_defaults. NF) _dirty_cachernr~setattr)r0Z attr_defaultsZ attr_valuesZ attributer.r!r!r$clear_cached_attrss  zDataSource.clear_cached_attrscCs"d|_|}|s|S||S)zDatasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. T)rrpersist_instance_data)r0Z return_valuer!r!r$get_datas zDataSource.get_dataTc Cs|r&tj|jjr&t||jdt|drbt t |d}| dd| ddd|i}n^dd|j ii}t|drt |d}|t kr||dd<t|d rt |d }|t kr||dd <t|dd <t |j|d <d |d d <t |d |d <d|d d <t|d<|||z"t|}tt||jd}Wnrtk r~}ztdt|WYdSd}~XYn:tk r}ztdt|WYdSd}~XYnX|jd} |d dd} tj!|jj"d} t#| d| | dd} | d| } tj$| r.tj%| } tj&| | dd| rX| | krXt'| t(| |dd|jd}t(|t)|dS)aPProcess and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. :param write_cache: boolean set True to persist obj.pkl when instance_link exists. @return True on successful write, False otherwise. Zobj_pklZ_crawled_metadatarqNrrr)Z meta_datargrfZ_docroz, (str, bool). is_default is a bool and it's true only if hostname is localhost and was returned by util.get_hostname() as a default. This is used to differentiate with a user-defined localhost hostname. Optionally return (None, False) when metadata_only is True and local-hostname data is not available. Z localdomainZ localhostFrNT.rzip-%srrez%s.%s)rQrWr_rrZget_fqdn_from_hostsfindr:rXrZis_ipv4_addressZ gethostbyaddrrKrr) r0ZfqdnZ resolve_ipZ metadata_onlyZ defdomainZdefhostZdomainraZtoksr`Z hosts_fqdnZlhostr!r!r$rsF         zDataSource.get_hostnamecCs|jj|dS)N)Z data_source)rtget_package_mirror_infor/r!r!r$r_sz"DataSource.get_package_mirror_info)source_event_typescCsPi}|D]B}|jD]2\}}||kr||s:t||<|||qq|Sr,)supported_update_eventsrFrWsetadd)r0rsupported_eventseventZ update_scopeZ update_eventsr!r!r$get_supported_eventsbs  zDataSource.get_supported_events)rr+c Cs||}|D]<\}}td|jddd|D|d|tffq|rl||}|rldStd|ddd|Dd S) aRefresh cached metadata if the datasource supports this event. The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @param source_event_types: List of EventTypes which may trigger a metadata update. @return True if the datasource did successfully update cached metadata due to source_event_type. z:Update datasource metadata and %s config due to events: %s, cSsg|] }|jqSr!r-.0rr!r!r$ sz;DataSource.update_metadata_if_supported..z _%s_configTz(Datasource %s not updated for events: %scSsg|] }|jqSr!r-rr!r!r$rsF) rrFrrr.rrrr)r0rrZscopeZmatched_eventsresultr!r!r$update_metadata_if_supportedos& z'DataSource.update_metadata_if_supportedcCsdSrr!)r0rvr!r!r$check_instance_idszDataSource.check_instance_idcCsV|dkr t}|dkrt}|D]4}|dkr*q||kr:|Std|||Sq|S)Nz%invalid dsmode '%s', using default=%s)DSMODE_NETWORK VALID_DSMODESrr)Z candidatesdefaultZvalid candidater!r!r$_determine_dsmodes  zDataSource._determine_dsmodecCsdSr,r!r/r!r!r$network_configszDataSource.network_configcCsdS)a(setup(is_new_instance) This is called before user-data and vendor-data have been processed. Unless the datasource has set mode to 'local', then networking per 'fallback' or per 'network_config' will have been written and brought up the OS at this point. Nr!)r0is_new_instancer!r!r$setups zDataSource.setupcCsdS)aactivate(cfg, is_new_instance) This is called before the init_modules will be called but after the user-data and vendor-data have been fully processed. The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. is_new_instance is a boolean indicating if this is a new instance. Nr!)r0Zcfgrr!r!r$activates zDataSource.activate)N)r!)T)F)FFF)NN)\r2r3r4rZdsmoderrwrrrrr'r6r9r8r7rbr __annotations__rrrrrrrBOOT_NEW_INSTANCEZBOOTZ BOOT_LEGACYZHOTPLUGrZdefault_update_eventsrrnr:rrrsr}Z_ci_pkl_versionrrr{rrr1boolrrrrrrrrrrrrpropertyrrrrrrrrrrrrrrrrrrrrrrrrrrr staticmethodrrrrr!r!r!r$rs    &  R<             H &   ) metaclasscCsg}|s |St|tr|St|ttfr4t|St|tr|D]>\}}t|tr^|g}t|ttfrF|D]}|rp||qpqF|Sr,)rIr: splitlineslistrrLrFrH)Z pubkey_datakeysZ_keynameZklistZpkeyr!r!r$rs    rr*c Cs"t|||}dd|D}t|kr&dnd} td| |t||D]\} } tjd| ddd | | fd | | f|d } zh| Ztd | | |||} | t j grd | | f| _ | t | fW5QRWSW5QRXWqBtk rttd| YqBXqBdd|}t|dS)NcSsg|]}t|qSr!r)rrr!r!r$rszfind_source..Znetworkrz#Searching for %s data source in: %sz search-%srr@zsearching for %s data from %szno %s data found from %s)nameZ descriptionmessageparentz%Seeing if we can get any data from %szfound %s data from %szGetting data from %s failedz4Did not find any data source, searched classes: (%s)r) list_sources DEP_NETWORKrrziprZReportEventStackrKrrrrrrrrrrr=)rvrtruZds_depscfg_listpkg_listZreporterds_listZds_namesrrclsZmyrepsmsgr!r!r$ find_sources4     .rc Csg}td||||D]j}t|}t||dg\}}|sJtd||D]2}t|} t| d} | |} | rN|| qqNq|S)zReturn a list of classes that have the same depends as 'depends' iterate through cfg_list, loading "DataSource*" modules and calling their "get_datasource_list". Return an ordered list of classes that match (if any) zLLooking for data source in: %s, via packages %s that matches dependencies %sZget_datasource_listzDCould not import %s. Does the DataSource exist and is it importable?) rrr Z"match_case_insensitive_module_name find_moduleerror import_modulerrN) rdependsrZsrc_listr)Zds_nameZm_locsZ _looked_locsZm_locmodZlisterZmatchesr!r!r$r s6    r system-uuid)fieldr+cCs*|sdSt|}|sdS||kSr)r Z read_dmi_datarG)rr"Z dmi_valuer!r!r$instance_id_matches_system_uuid/s  r#cCsl|st}|st}|tkr(|tkr$|S|StD]*\}}|\}}||r0||r0|Sq0|tkrh|S|S)z@Lookup the canonical cloud-id for a given cloud_name and region.)rCLOUD_ID_REGION_PREFIX_MAPrFrJ)rrrprefixZ cloud_id_testrZ valid_cloudr!r!r$r=s rTcCsj|sdSt|tr|St|tr*t|St|trV|dkrNt|dddStdtdt |dS)aLdata: a loaded object (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: None: return None string: return string list: return data the list is then processed in UserDataProcessor dict: return convert_vendordata(data.get('cloud-init')) NTz cloud-initF)recursez'vendordata['cloud-init'] cannot be dictz$Unknown data type for vendordata: %s) rIr:r rDrErLconvert_vendordatarWrr)datar&r!r!r$r'Ps     r'c@s eZdZdS)BrokenMetadataNr>r!r!r!r$r)hsr)cCs4g}t|}|D]\}}|t|kr||q|Sr,)rrH)rrZret_listZdepsetrZdepsr!r!r$list_from_dependsss    r*)rZfnamer+cCstzt|}Wn$tk r2ttd|YdSXztj||dddWn$tk rnttd|YdSXdS)z[Use pickle to serialize Datasource to a file as a cache. :return: True on success zFailed pickling datasource %sFwb)Zomoderz Failed pickling datasource to %sT)pickledumpsrrrrr)rZr+Z pk_contentsr!r!r$r|sr)r+r+c Csd}ztj|dd}Wn<tk rR}ztj|rBtd||W5d}~XYnX|s\dSz t |WSt k r~YdStk rt td|YdSXdS)zBUse pickle to deserialize a instance Datasource from a cache file.NF)decodezfailed loading pickle in %s: %sz#Failed loading pickled blob from %s) rZ load_filerrr[isfilerrr.rr;r)r+Zpickle_contentsrr!r!r$pkl_loads   r2cCst}td|}td|}td|}|p6|p6|}|p>|}|rv|d}tjd|d|dd|d d |r|dr|dSd S) zCheck if command line argument for this datasource was passed Passing by command line overrides runtime datasource detection z ds=([^\s;]+)zci\.ds=([^\s;]+)zci\.datasource=([^\s;]+)rez7Defining the datasource on the commandline using ci.ds=z or ci.datasource=z23.2zUse ds=z instead) deprecatedZdeprecated_versionZ extra_messager@)rZ get_cmdlineresearchgroupstripZ deprecate)r(Z ds_parse_0Z ds_parse_1Z ds_parse_2r)r3rwr!r!r$rs        r)r@r!)r!)T)UabcrDrrr.r4 collectionsrenumrrtypingrrrrr Z cloudinitr r r r ZloggingrrrryrZcloudinit.atomic_helperrZcloudinit.distrosrZcloudinit.eventrrZcloudinit.filtersrZcloudinit.helpersrZcloudinit.persistencerZcloudinit.reportingrZDSMODE_DISABLEDZ DSMODE_LOCALrZ DSMODE_PASSrZDEP_FILESYSTEMrZ DS_PREFIXrZREDACT_SENSITIVE_VALUErrrZ getLoggerr2rr$r'rr;r=r?rMr\r]r_ABCMetarrr:rrrr#rr'IOErrorr)r*rr2rr!r!r!r$ s                #    !%