U 0h>@sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZmZmZmZmZmZddlmZmZmZmZmZmZmZddlmZddlmZddlm Z dd l!m"Z"dd l#m$Z$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+ddl,m-Z-dZ.dZ/dZ0dZ1e.e/e0gZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:e;e<Z=dddfdddfd d!dfd"Z>e Gd#d$d$e Z?Gd%d&d&e Z@Gd'd(d(eAZBGd)d*d*eAZCGd+d,d,eAZDd^d/d0ZEe7fd1d2ZFed3d4d5d6d7gZGed8d9d:gZHGd;d<dZKd?d@ZLeeKeMfdAdBdCZNdDdEZOd_eMePdGdHdIZQdJdKZRd`dMdNZSGdOdPdPeTZUdQdRZVeKeMePdSdTdUZWeMeeKdVdWdXZXeMdAdYdZZYeMeMd[d\d]ZZdS)aN) namedtuple)Enumunique)AnyDictList NamedTupleOptionalTupleUnion) atomic_helperdmiimporter lifecyclenet performance type_utils) user_data)util) write_json)Distro) EventScope EventType) launch_index)Paths)CloudInitPickleMixin)eventsZdisabledlocalrpassZ FILESYSTEMNETWORK DataSourcez|EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.zredacted for non-root user cloud-nameZ_unsetunknownz aws-chinacCs|dkSNZawscr$r$Gr(zaws-govcCs|dkSr#r$r%r$r$r'r(Hr)z azure-chinacCs|dkS)NZazurer$r%r$r$r'r(Ir))zcn-zus-gov-Zchinac@s2eZdZdZdZdZdZdZdZe ddd Z d S) NetworkConfigSourcezb Represents the canonical list of network config sources that cloud-init knows about. ZcmdlinedsZ system_cfgZfallbackZ initramfsreturncCs|jSNvalueselfr$r$r'__str__ZszNetworkConfigSource.__str__N) __name__ __module__ __qualname____doc__CMD_LINEDS SYSTEM_CFGZFALLBACK INITRAMFSstrr3r$r$r$r'r*Msr*c@s&eZdZdZdZdZedddZdS)NicOrderzRepresents ways to sort NICsZmacZnic_namer,cCs|jSr.r/r1r$r$r'r3dszNicOrder.__str__N)r4r5r6r7ZMACZNIC_NAMEr<r3r$r$r$r'r=^sr=c@seZdZdZdS)DatasourceUnpickleUserDataErrorzERaised when userdata is unable to be unpickled due to python upgradesNr4r5r6r7r$r$r$r'r>hsr>c@s eZdZdS)DataSourceNotFoundExceptionNr4r5r6r$r$r$r'r@lsr@c@seZdZdZdS)InvalidMetaDataExceptionz8Raised when metadata is broken, unavailable or disabled.Nr?r$r$r$r'rBpsrBr$c Cst|}g}g}|D]\}}|r4|d|}n|}||ksP||krZ||t|tr|dr|||dd||<t|t rt |||} | | d| | d| ||<qt ||d<t ||d<|S)zProcess all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @return Dict copy of processed metadata. /zci-b64:rCbase64_encoded_keyssensitive_keys)copydeepcopyitemslowerappend isinstancer< startswithreplacedictprocess_instance_metadataextendpopsorted) metadatakey_pathrFmd_copyrEZ sens_keyskeyvalZ sub_key_pathZ return_valr$r$r'rPts6         rPcCs|dgs|St|}|dD]V}|d}|}|D].}||kr:t||tr:||dkr:||}q:||kr$|||<q$|S)zRedact any sensitive keys from to provided metadata dictionary. Replace any keys values listed in 'sensitive_keys' with redact_value. rFrD)getrGrHsplitrLrO)rTZ redact_valuerVrU path_partsobjpathr$r$r'redact_sensitive_keyss"       r_ URLParamsZmax_wait_secondsZtimeout_secondsZ num_retriessec_between_retriesDataSourceHostnamehostname is_defaultc@s*eZdZUdZeed<eed<eed<dS)HotplugRetrySettingsz in secondsZ force_retryZ sleep_periodZ sleep_totalN)r4r5r6r7bool__annotations__intr$r$r$r'res rec @seZdZUeZdZdZdZee e d<dZ dZ dZ eeee fe d<ejejejejfZeedfe d<dZd Zd Zd Zejejejej ej!hiZ"ejejhiZ#d e$fd e$fdifddddddf Z%eee e&fdfe d<dZ'dZ(ee dfe d<dZ)e*dddZ+dZ,ee e d<d Z-d}e.e/dddZ0e1dddd Z2d!d"Z3e4d#d$d%Z5e4d#d&d'Z6d(d)Z7d*d+Z8d~d-d.Z9e:j;d/d0d1e4d#d2d3Zd9d:Z?dd;d<Z@d=d>ZAd?d@ZBeCdAdBZDeCdCdDZEdEdFZFeCdGdHZGdIdJZHeCdKdLZIdMdNZJdOdPZKdQdRZLdSdTZMdUdVZNdWdXZOdYdZZPd[d\ZQd]d^ZRd_d`ZSeCdadbZTeCdcddZUdedfZVddgdhZWdidjZXeYedkdldmZZeYee4dndodpZ[dqdrZ\dsdtZ]e^ddudvZ_eCdwdxZ`dydzZad{d|ZbdS)r z en_US.UTF-8Z_undefN _cloud_name_crawled_metadata.network_config_sourcesrY  ec2_metadata network_jsonrT)userdataN) userdata_rawN) vendordataN)vendordata_rawN) vendordata2N)vendordata2_rawNcached_attr_defaultsF) Zcombined_cloud_config merged_cfgmerged_system_cfgzsecurity-credentialsrq user-datarrs vendor-datazds/vendor_datasensitive_metadata_keysrextra_hotplug_udev_rules)distropathscCs||_||_||_d|_i|_d|_d|_d|_d|_d|_ d|_ t |_ t |_ t|jd|jfi|_|jsri|_|st|j|_n||_dS)NZ datasource)sys_cfgr~rrqrTrrrsrurtrvmetadata_addressUNSETrprorZget_cfg_by_pathdsnameds_cfgudZUserDataProcessorud_proc)r2rr~rrr$r$r'__init__Ts.zDataSource.__init__)ci_pkl_versionr-cCsdddtddtdddtdddd }|D]\}}t||s,t|||q,t|dsft|dddt|dr|jdk rzt|jWn6tk r}zt d |t |W5d}~XYnXdS) z(Perform deserialization fixes for Paths.NFr) rj_platform_type _subplatformror}rrpskip_hotplug_detectrurvhotplug_retry_settingscheck_if_fallback_is_allowedcSsdSNFr$r$r$r$r'r(r)z&DataSource._unpickle..rqz:Unable to unpickle datasource: %s. Ignoring current cache.) rrerIhasattrsetattrrqr<AttributeErrorLOGdebugr>)r2rZexpected_attrsrWr0er$r$r' _unpicklens4    zDataSource._unpicklecCs t|Sr.robj_namer1r$r$r'r3szDataSource.__str__r,cCsdS)z#Check if running on this datasourceTr$r1r$r$r' ds_detectszDataSource.ds_detectcCsN|jtkr$td|dS|jdg|jgkrJtd|dSdS)aJOverride if either: - only a single datasource defined (nothing to fall back to) - command line argument is used (ci.ds=OpenStack) Note: get_cmdline() is required for the general case - when ds-identify does not run, _something_ needs to detect the kernel command line definition. z6Kernel command line set to use a single datasource %s.TZdatasource_listz2Datasource list set to use a single datasource %s.F)rrJ parse_cmdlinerrrrZr1r$r$r'override_ds_detects zDataSource.override_ds_detectcCs@|r|S|r,td||Std|dSdS)z&Overrides runtime datasource detectionz Detected %szDid not detect %sFN)r _get_datarrrr1r$r$r'_check_and_get_datas zDataSource._check_and_get_datacCs|j}|}|j}|d}ddg||t|j|j|j|j|j|dd|dd|dd|j||d|||d d|||d d |j|j |d |d d iS)z2Return a dictionary of standardized metadata keys.sys_infov1 subplatformZdistrrnpythonunameplatformvariant)Z _beta_keysavailability-zoneavailability_zonecloud_idr! cloud_namer~Zdistro_versionZdistro_releaserZpublic_ssh_keysZpython_version instance-id instance_idZkernel_releaselocal-hostnamelocal_hostnamemachineregionrZsystem_platformr) get_hostnamercget_instance_idrcanonical_cloud_idrr platform_typeget_public_ssh_keysr)r2 instance_datarrrZsysinfor$r$r'_get_standardized_metadatasB      z%DataSource._get_standardized_metadatar$cCsL|js dS|r|}n|j}|D]\}}t||rt|||q|sHd|_dS)zReset any cached metadata attributes to datasource defaults. @param attr_defaults: Optional tuple of (attr, value) pairs to set instead of cached_attr_defaults. NF) _dirty_cacherwrr)r2Z attr_defaultsZ attr_valuesZ attributer0r$r$r'clear_cached_attrss  zDataSource.clear_cached_attrszGetting metadataalways)Zlog_modecCs"d|_|}|s|S||S)zDatasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. T)rrpersist_instance_data)r2Z return_valuer$r$r'get_datas zDataSource.get_dataTc Cs`|r&tj|jjr&t||jd|jdk r^t |j}| dd| ddd|i}n>dd|j ii}|j t kr|j |dd<|jt kr|j|dd<t|dd <t |j|d <d |d d <t |d |d <d |d d <t|d<|||z"t|}tt||jd}Wnrtk rZ}ztdt|WYdSd}~XYn:tk r}ztdt|WYdSd}~XYnX|j d}|d!dd}tj"|jj#d} t$| d||dd} | d|} tj%| r tj&| } tj'| | dd| r4| | kr4t(| t)||dd|j d} t)| t*|dS)aPProcess and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. :param write_cache: boolean set True to persist obj.pkl when instance_link exists. @return True on successful write, False otherwise. Zobj_pklNrzr{r+Z meta_datarproZ_docrxz, (str, bool). is_default is a bool and it's true only if hostname is localhost and was returned by util.get_hostname() as a default. This is used to differentiate with a user-defined localhost hostname. Optionally return (None, False) when metadata_only is True and local-hostname data is not available. Z localdomainZ localhostFrNT.rzip-%srrnz%s.%s)rTrZrbrrZget_fqdn_from_hostsfindr<r[rZis_ipv4_addressZ gethostbyaddrrNrr) r2ZfqdnZ resolve_ipZ metadata_onlyZ defdomainZdefhostZdomainrdZtoksrcZ hosts_fqdnZlhostr$r$r'rFsF         zDataSource.get_hostnamecCs|jj|dS)N)Z data_source)r~get_package_mirror_infor1r$r$r'rsz"DataSource.get_package_mirror_info)source_event_typescCsPi}|D]B}|jD]2\}}||kr||s:t||<|||qq|Sr.)supported_update_eventsrIrZsetadd)r2rsupported_eventseventZ update_scopeZ update_eventsr$r$r'get_supported_eventss  zDataSource.get_supported_events)rr-c Cs||}|D]<\}}td|jddd|D|d|tffq|rl||}|rldStd|ddd|Dd S) aRefresh cached metadata if the datasource supports this event. The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @param source_event_types: List of EventTypes which may trigger a metadata update. @return True if the datasource did successfully update cached metadata due to source_event_type. z:Update datasource metadata and %s config due to events: %s, cSsg|] }|jqSr$r/.0rr$r$r' sz;DataSource.update_metadata_if_supported..z _%s_configTz(Datasource %s not updated for events: %scSsg|] }|jqSr$r/rr$r$r'rsF) rrIrrr0rrrr)r2rrZscopeZmatched_eventsresultr$r$r'update_metadata_if_supporteds& z'DataSource.update_metadata_if_supportedcCsdSrr$)r2rr$r$r'check_instance_idszDataSource.check_instance_idcCsdS)acheck_if_fallback_is_allowed() Checks if a cached ds is allowed to be restored when no valid ds is found in local mode by checking instance-id and searching valid data through ds list. @return True if a ds allows fallback, False otherwise. Fr$r1r$r$r'rsz'DataSource.check_if_fallback_is_allowedcCsV|dkr t}|dkrt}|D]4}|dkr*q||kr:|Std|||Sq|S)Nz%invalid dsmode '%s', using default=%s)DSMODE_NETWORK VALID_DSMODESrr)Z candidatesdefaultZvalid candidater$r$r'_determine_dsmodes  zDataSource._determine_dsmodecCsdSr.r$r1r$r$r'network_configszDataSource.network_configcCsdS)a(setup(is_new_instance) This is called before user-data and vendor-data have been processed. Unless the datasource has set mode to 'local', then networking per 'fallback' or per 'network_config' will have been written and brought up the OS at this point. Nr$)r2is_new_instancer$r$r'setups zDataSource.setupcCsdS)aactivate(cfg, is_new_instance) This is called before the init_modules will be called but after the user-data and vendor-data have been fully processed. The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. is_new_instance is a boolean indicating if this is a new instance. Nr$)r2Zcfgr r$r$r'activates zDataSource.activate)N)r$)T)F)FFF)NN)cr4r5r6rZdsmoderrrir r<rgrrrjr rr*r8r;r:r9rkr rrrrrrrBOOT_NEW_INSTANCEZBOOTZ BOOT_LEGACYZHOTPLUGrZdefault_update_eventsrrwrrr|rrerr}Z_ci_pkl_versionrrrrhrr3rfrrrrrrZtimedrrrrrrrpropertyrrrrrrrrrrrrrrrrrrrrrrrrrr staticmethodr r r r r$r$r$r'r s      %&   L<           H &    ) metaclasscCsg}|s |St|tr|St|ttfr4t|St|tr|D]>\}}t|tr^|g}t|ttfrF|D]}|rp||qpqF|Sr.)rLr< splitlineslistrrOrIrK)Z pubkey_datakeysZ_keynameZklistZpkeyr$r$r'rs    rr,c Cs"t|||}dd|D}t|kr&dnd} td| |t||D]\} } tjd| ddd | | fd | | f|d } zh| Ztd | | |||} | t j grd | | f| _ | t | fW5QRWSW5QRXWqBtk rttd| YqBXqBdd|}t|dS)NcSsg|]}t|qSr$r)rrr$r$r'r'szfind_source..Znetworkrz#Searching for %s data source in: %sz search-%sr rCzsearching for %s data from %szno %s data found from %s)nameZ descriptionmessageparentz%Seeing if we can get any data from %szfound %s data from %szGetting data from %s failedz4Did not find any data source, searched classes: (%s)r) list_sources DEP_NETWORKrrziprZReportEventStackrNrrrrrrrrrrr@)rr~rZds_depscfg_listpkg_listZreporterds_listZds_namesrrclsZmyrepsmsgr$r$r' find_source#s4     .r!c Csg}td||||D]j}t|}t||dg\}}|sJtd||D]2}t|} t| d} | |} | rN|| qqNq|S)zReturn a list of classes that have the same depends as 'depends' iterate through cfg_list, loading "DataSource*" modules and calling their "get_datasource_list". Return an ordered list of classes that match (if any) zLLooking for data source in: %s, via packages %s that matches dependencies %sZget_datasource_listzDCould not import %s. Does the DataSource exist and is it importable?) rrrZ"match_case_insensitive_module_name find_moduleerror import_modulegetattrrQ) rdependsrZsrc_listr+Zds_nameZm_locsZ _looked_locsZm_locmodZlisterZmatchesr$r$r'rDs6    r system-uuid)fieldr-cCs*|sdSt|}|sdS||kSr)r Z read_dmi_datarJ)rr)Z dmi_valuer$r$r'instance_id_matches_system_uuidhs  r*cCsl|st}|st}|tkr(|tkr$|S|StD]*\}}|\}}||r0||r0|Sq0|tkrh|S|S)z@Lookup the canonical cloud-id for a given cloud_name and region.)rCLOUD_ID_REGION_PREFIX_MAPrIrM)rrrprefixZ cloud_id_testrZ valid_cloudr$r$r'rvs rTcCsj|sdSt|tr|St|tr*t|St|trV|dkrNt|dddStdtdt |dS)aLdata: a loaded object (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: None: return None string: return string list: return data the list is then processed in UserDataProcessor dict: return convert_vendordata(data.get('cloud-init')) NTz cloud-initF)recursez'vendordata['cloud-init'] cannot be dictz$Unknown data type for vendordata: %s) rLr<rrGrHrOconvert_vendordatarZrr)datar-r$r$r'r.s     r.c@s eZdZdS)BrokenMetadataNrAr$r$r$r'r0sr0cCs4g}t|}|D]\}}|t|kr||q|Sr.)rrK)r&rZret_listZdepsetrZdepsr$r$r'list_from_dependss    r1)r]fnamer-cCstzt|}Wn$tk r2ttd|YdSXztj||dddWn$tk rnttd|YdSXdS)z[Use pickle to serialize Datasource to a file as a cache. :return: True on success zFailed pickling datasource %sFwb)Zomoderz Failed pickling datasource to %sT)pickledumpsrrrrr)r]r2Z pk_contentsr$r$r'rsr)r2r-c Csd}zt|}Wn<tk rN}ztj|r>td||W5d}~XYnX|sXdSz t |WSt k rzYdStk rt td|YdSXdS)zBUse pickle to deserialize a instance Datasource from a cache file.Nzfailed loading pickle in %s: %sz#Failed loading pickled blob from %s) rZload_binary_filerrr^isfilerrr5rr>r)r2Zpickle_contentsrr$r$r'pkl_loads   r8cCs ttS)zCheck if command line argument for this datasource was passed Passing by command line overrides runtime datasource detection )parse_cmdline_or_dmirZ get_cmdliner$r$r$r'rsr)inputr-cCstd|}td|}td|}|p.|p.|}|p6|}|rn|d}tjd|d|dd|d d |r|dr|dSd S) Nz(?:^|\s)ds=([^\s;]+)z(?:^|\s)ci\.ds=([^\s;]+)z (?:^|\s)ci\.datasource=([^\s;]+)rnz8Defining the datasource on the command line using ci.ds=z or ci.datasource=z23.2zUse ds=z instead) deprecatedZdeprecated_versionZ extra_messagerC)researchgroupstriprZ deprecate)r:Z ds_parse_0Z ds_parse_1Z ds_parse_2r+r;rr$r$r'r9s       r9)rCr$)r()T)[abcrGrZloggingrr5r< collectionsrenumrrtypingrrrrr r r Z cloudinitr r rrrrrrrrZcloudinit.atomic_helperrZcloudinit.distrosrZcloudinit.eventrrZcloudinit.filtersrZcloudinit.helpersrZcloudinit.persistencerZcloudinit.reportingrZDSMODE_DISABLEDZ DSMODE_LOCALrZ DSMODE_PASSrZDEP_FILESYSTEMrZ DS_PREFIXrZREDACT_SENSITIVE_VALUErrrZ getLoggerr4rr+r*r=rr>r@rBrPr_r`rbreABCMetar rr<r!rrfr*rr.IOErrorr0r1rr8rr9r$r$r$r' s $$               #  @ !%