PNG  IHDRxsBIT|d pHYs+tEXtSoftwarewww.inkscape.org<,tEXtComment File Manager

File Manager

Path: /opt/cloudlinux/venv/lib/python3.11/site-packages/lvestats/plugins/generic/

Viewing File: dbsaver_x60.py

# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

import logging
import sys
import time

from sqlalchemy import insert
from sqlalchemy.exc import OperationalError
from sqlalchemy.sql import select
from sqlalchemy.sql.expression import func

from lvestats.core.plugin import LveStatsPlugin, LveStatsPluginTerminated
from lvestats.lib import dbengine
from lvestats.lib.commons.func import get_chunks, reboot_lock
from lvestats.lib.commons.progress import ProgressCallback
from lvestats.lib.lveinfolib import FIELD_TO_TABLE_COLUMN, HistoryShow
from lvestats.orm.history import history, history_x60

AGGREGATE_PERIOD = 60 * 60  # one hour


class DBSaverX60(LveStatsPlugin):
    FIELDS = [
        'ID',
        'aCPU',
        'lCPU',
        'CPUf',
        'aEP',
        'lEP',
        'EPf',
        'aVMem',
        'lVMem',
        'VMemF',
        'aPMem',
        'lPMem',
        'PMemF',
        'aNproc',
        'lNproc',
        'NprocF',
        'aIO',
        'lIO',
        'IOf',
        'IOPSf',
        'lIOPS',
        'aIOPS',
    ]

    def __init__(self):
        self.log = logging.getLogger('plugin.DBSaverX60')
        self.now = 0  # This changes in MainLoop
        self.config = None
        self.period = AGGREGATE_PERIOD  # once an hour
        self.server_id = 'localhost'
        self.verbose_ = False
        self.execute_timeout = 20
        self.progress = ProgressCallback(time_change_step=0.1)

    def set_config(self, _config):
        self.config = _config

    def aggregate_db_data_by_hours(self, utc_from, utc_to):
        if self.verbose_:
            self.progress(0)
        utc_from_ = int(utc_from) // int(AGGREGATE_PERIOD) * int(AGGREGATE_PERIOD)
        if utc_from_ != utc_from:
            utc_from_ += AGGREGATE_PERIOD

        if utc_to - utc_from_ > AGGREGATE_PERIOD:
            self.log.debug("Data aggregation from %s to %s started.", utc_from_, utc_to)
            for aggr_cycle, hour_from in enumerate(range(utc_from_, int(utc_to), AGGREGATE_PERIOD), 1):
                hour_to = hour_from + AGGREGATE_PERIOD
                if hour_to > utc_to:
                    break
                history_show = HistoryShow(
                    dbengine=self.engine,
                    period_from=hour_from + 1,  # "+1" for exclude "hour_from" timestamp
                    period_to=hour_to,
                    show_columns=self.FIELDS,
                    server_id=self.server_id,
                    log=self.log,
                )
                rows = history_show.proceed()
                mass_insert_values = []
                for row in rows:
                    one_insert_values = {
                        'server_id': self.server_id,
                        'time': history_show.time_count,
                        'created': hour_to,
                    }
                    one_insert_values.update(
                        {FIELD_TO_TABLE_COLUMN[field_]: row[indx_] for indx_, field_ in enumerate(self.FIELDS)}
                    )
                    # correct cpu
                    one_insert_values['cpu'] *= 100
                    one_insert_values['cpu_limit'] *= 100
                    mass_insert_values.append(one_insert_values)

                with reboot_lock():
                    conn_ = self.engine.connect()
                    trans = conn_.begin()
                    try:
                        if mass_insert_values:
                            for chunk in get_chunks(mass_insert_values):
                                conn_.execute(insert(history_x60), chunk)
                        else:
                            conn_.execute(
                                insert(history_x60), {'server_id': self.server_id, 'created': hour_to, 'id': 0}
                            )
                        if self.verbose_:  # show progress
                            progress_ = 100 - (utc_to - hour_from) * 100 // (utc_to - utc_from_)
                            self.progress(progress_)
                    except OperationalError as oe:
                        trans.rollback()
                        self.log.error('Can not write aggregated data to database: %s', str(oe))
                        if self.verbose_:
                            print(f' Warning: {str(oe)[:255]}...')
                    except LveStatsPluginTerminated as exc:
                        trans.rollback()
                        conn_.close()
                        self.log.debug("Plugin is terminated.")
                        raise LveStatsPluginTerminated() from exc
                    except TypeError as oe:
                        trans.rollback()
                        conn_.close()
                        msg = str(oe)
                        # try fix incorrect database records after migration
                        if "*=: 'NoneType' and 'int'" not in msg:
                            raise
                        self.log.warning(msg)
                        from lvestats.lib.dbengine import fix_db  # pylint: disable=import-outside-toplevel

                        fix_db(self.engine, self.config, from_timestmp=hour_from, to_timestamp=hour_to, log_=self.log)
                        return
                    else:
                        trans.commit()
                        conn_.close()

                work_time = time.time() - self.now  # calculate plugin working time
                if (
                    self.execute_timeout > 0 and self.execute_timeout - work_time < work_time / aggr_cycle + 1
                ):  # continue if can't finish in time
                    self.log.debug('Data was aggregated from %s to %s in %s seconds', utc_from_, hour_to, work_time)
                    return
            self.log.debug("Data aggregation done.")
        else:
            self.log.debug("Nothing to aggregate.")
        if self.verbose_:
            self.progress(100, force=True)
            self.progress.stop()

    def execute(self, lve_data):
        self.log.debug("Plugin executed")
        self.server_id = self.config.get('server_id', 'localhost')
        dt_x1_max, dt_x1_min, dt_x60_max, dt_x60_min = self.get_max_min_ts()

        self.aggregate_new(dt_x1_max, dt_x1_min, dt_x60_max)
        self.aggregate_migrated(dt_x1_min, dt_x60_min)

    def aggregate_migrated(self, dt_x1_min, dt_x60_min):
        self.log.debug("Aggregating migrated")
        if dt_x1_min is not None and dt_x60_min is not None and dt_x1_min + AGGREGATE_PERIOD < dt_x60_min:
            self.aggregate_db_data_by_hours(dt_x60_min - AGGREGATE_PERIOD, dt_x60_min)

    def aggregate_new(self, dt_x1_max, dt_x1_min, dt_x60_max):
        if dt_x1_max:
            if not dt_x60_max:
                dt_x60_max = dt_x1_min  # from minimum timestamp
            if self.verbose_:
                print(
                    f'[lve-stats]: Start aggregating data from "{dt_x60_max}" '
                    f'to "{dt_x1_max}"; to skip press Ctrl+C ...'
                )
            # correct execute period
            if dt_x1_max - dt_x60_max > AGGREGATE_PERIOD * 2:  # run plugin often if need aggregate more than one hour
                self.period = 60  # FIXME: This looks like mistake?
            else:
                self.period = AGGREGATE_PERIOD
            self.aggregate_db_data_by_hours(dt_x60_max, dt_x1_max)

    def get_max_min_ts(self):
        conn_ = self.engine.connect()
        try:
            # get last created
            query = select([func.max(history.created), func.min(history.created)]).where(
                self.server_id == history.server_id
            )
            time_start = time.time()
            row = conn_.execute(query)
            dt_x1_max, dt_x1_min = row.fetchone()
            self.log.debug(
                '%s; query time: %s',
                str(query.compile(compile_kwargs={"literal_binds": True})).replace('\n', ' '),
                time.time() - time_start,
            )

            # get last created_max
            query = select([func.max(history_x60.created), func.min(history_x60.created)]).where(
                self.server_id == history_x60.server_id
            )
            row = conn_.execute(query)
            dt_x60_max, dt_x60_min = row.fetchone()
            self.log.debug(
                '%s; query time: %s',
                str(query.compile(compile_kwargs={"literal_binds": True})).replace('\n', ' '),
                time.time() - time_start,
            )
        except LveStatsPluginTerminated as exc:
            conn_.close()
            self.log.debug("Plugin is terminated.")
            raise LveStatsPluginTerminated() from exc
        else:
            conn_.close()

        return dt_x1_max, dt_x1_min, dt_x60_max, dt_x60_min


def main(argv_, config_, db_engine=None):
    lve_data = {}
    verbose_ = '--verbose' in argv_
    if db_engine is None:
        db_engine = dbengine.make_db_engine(config_, debug=False)
    if 'init' in argv_:
        if verbose_:
            print(f'[lve-stats]: creating table "{history_x60.__tablename__}"...')
        with db_engine.begin() as conn_:
            history_x60.__table__.create(bind=conn_, checkfirst=True)  # create table if exist
    if 'aggregate' in argv_:
        # configure plugin
        plugin_instance = DBSaverX60()
        plugin_instance.config = config_  # use specific config
        plugin_instance.set_db_engine(db_engine)
        plugin_instance.execute_timeout = 2 * 60 * 60  # two hours
        plugin_instance.verbose_ = verbose_
        plugin_instance.now = time.time()

        plugin_instance.execute(lve_data)  # generate aggregate data


# for initial from command line
# create table if exist and generate aggregate data
if __name__ == '__main__' and len(sys.argv) >= 2:
    from lvestats.lib import config

    try:
        cnf = config.read_plugin_config(config.read_config(), plugin=DBSaverX60.__name__)
        main(argv_=sys.argv[1:], config_=cnf)
    except KeyboardInterrupt:
        print('Stopped')
    except dbengine.MakeDbException as e:
        print(str(e))
    except config.ConfigError as ce:
        ce.log_and_exit()
b IDATxytVսϓ22 A@IR :hCiZ[v*E:WũZA ^dQeQ @ !jZ'>gsV仿$|?g)&x-EIENT ;@xT.i%-X}SvS5.r/UHz^_$-W"w)Ɗ/@Z &IoX P$K}JzX:;` &, ŋui,e6mX ԵrKb1ԗ)DADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADADA݀!I*]R;I2$eZ#ORZSrr6mteffu*((Pu'v{DIߔ4^pIm'77WEEE;vƎ4-$]'RI{\I&G :IHJ DWBB=\WR޽m o$K(V9ABB.}jѢv`^?IOȅ} ڶmG}T#FJ`56$-ھ}FI&v;0(h;Б38CӧOWf!;A i:F_m9s&|q%=#wZprrrla A &P\\СC[A#! {olF} `E2}MK/vV)i{4BffV\|ۭX`b@kɶ@%i$K z5zhmX[IXZ` 'b%$r5M4º/l ԃߖxhʔ)[@=} K6IM}^5k㏷݆z ΗÿO:gdGBmyT/@+Vɶ纽z񕏵l.y޴it뭷zV0[Y^>Wsqs}\/@$(T7f.InݺiR$푔n.~?H))\ZRW'Mo~v Ov6oԃxz! S,&xm/yɞԟ?'uaSѽb,8GלKboi&3t7Y,)JJ c[nzӳdE&KsZLӄ I?@&%ӟ۶mSMMњ0iؐSZ,|J+N ~,0A0!5%Q-YQQa3}$_vVrf9f?S8`zDADADADADADADADADAdqP,تmMmg1V?rSI꒟]u|l RCyEf٢9 jURbztѰ!m5~tGj2DhG*{H9)꒟ר3:(+3\?/;TUݭʴ~S6lڧUJ*i$d(#=Yݺd{,p|3B))q:vN0Y.jkק6;SɶVzHJJЀ-utѹսk>QUU\޲~]fFnK?&ߡ5b=z9)^|u_k-[y%ZNU6 7Mi:]ۦtk[n X(e6Bb."8cۭ|~teuuw|ήI-5"~Uk;ZicEmN/:]M> cQ^uiƞ??Ңpc#TUU3UakNwA`:Y_V-8.KKfRitv޲* 9S6ֿj,ՃNOMߤ]z^fOh|<>@Å5 _/Iu?{SY4hK/2]4%it5q]GGe2%iR| W&f*^]??vq[LgE_3f}Fxu~}qd-ږFxu~I N>\;͗O֊:̗WJ@BhW=y|GgwܷH_NY?)Tdi'?խwhlmQi !SUUsw4kӺe4rfxu-[nHtMFj}H_u~w>)oV}(T'ebʒv3_[+vn@Ȭ\S}ot}w=kHFnxg S 0eޢm~l}uqZfFoZuuEg `zt~? b;t%>WTkķh[2eG8LIWx,^\thrl^Ϊ{=dž<}qV@ ⠨Wy^LF_>0UkDuʫuCs$)Iv:IK;6ֲ4{^6եm+l3>݆uM 9u?>Zc }g~qhKwڭeFMM~pМuqǿz6Tb@8@Y|jx](^]gf}M"tG -w.@vOqh~/HII`S[l.6nØXL9vUcOoB\xoǤ'T&IǍQw_wpv[kmO{w~>#=P1Pɞa-we:iǏlHo׈꒟f9SzH?+shk%Fs:qVhqY`jvO'ρ?PyX3lх]˾uV{ݞ]1,MzYNW~̈́ joYn}ȚF߾׮mS]F z+EDxm/d{F{-W-4wY듏:??_gPf ^3ecg ҵs8R2מz@TANGj)}CNi/R~}c:5{!ZHӋӾ6}T]G]7W6^n 9*,YqOZj:P?Q DFL|?-^.Ɵ7}fFh׶xe2Pscz1&5\cn[=Vn[ĶE鎀uˌd3GII k;lNmشOuuRVfBE]ۣeӶu :X-[(er4~LHi6:Ѻ@ԅrST0trk%$Č0ez" *z"T/X9|8.C5Feg}CQ%͞ˣJvL/?j^h&9xF`њZ(&yF&Iݻfg#W;3^{Wo^4'vV[[K';+mӍִ]AC@W?1^{එyh +^]fm~iԵ]AB@WTk̏t uR?l.OIHiYyԶ]Aˀ7c:q}ힽaf6Z~қm(+sK4{^6}T*UUu]n.:kx{:2 _m=sAߤU@?Z-Vކеz왍Nэ{|5 pڶn b p-@sPg]0G7fy-M{GCF'%{4`=$-Ge\ eU:m+Zt'WjO!OAF@ik&t݆ϥ_ e}=]"Wz_.͜E3leWFih|t-wZۍ-uw=6YN{6|} |*={Ѽn.S.z1zjۻTH]흾 DuDvmvK.`V]yY~sI@t?/ϓ. m&["+P?MzovVЫG3-GRR[(!!\_,^%?v@ҵő m`Y)tem8GMx.))A]Y i`ViW`?^~!S#^+ѽGZj?Vģ0.))A꨷lzL*]OXrY`DBBLOj{-MH'ii-ϰ ok7^ )쭡b]UXSְmռY|5*cֽk0B7镹%ڽP#8nȎq}mJr23_>lE5$iwui+ H~F`IjƵ@q \ @#qG0".0" l`„.0! ,AQHN6qzkKJ#o;`Xv2>,tێJJ7Z/*A .@fفjMzkg @TvZH3Zxu6Ra'%O?/dQ5xYkU]Rֽkق@DaS^RSּ5|BeHNN͘p HvcYcC5:y #`οb;z2.!kr}gUWkyZn=f Pvsn3p~;4p˚=ē~NmI] ¾ 0lH[_L hsh_ғߤc_њec)g7VIZ5yrgk̞W#IjӪv>՞y睝M8[|]\շ8M6%|@PZڨI-m>=k='aiRo-x?>Q.}`Ȏ:Wsmu u > .@,&;+!!˱tﭧDQwRW\vF\~Q7>spYw$%A~;~}6¾ g&if_=j,v+UL1(tWake:@Ș>j$Gq2t7S?vL|]u/ .(0E6Mk6hiۺzښOrifޱxm/Gx> Lal%%~{lBsR4*}{0Z/tNIɚpV^#Lf:u@k#RSu =S^ZyuR/.@n&΃z~B=0eg뺆#,Þ[B/?H uUf7y Wy}Bwegל`Wh(||`l`.;Ws?V@"c:iɍL֯PGv6zctM̠':wuW;d=;EveD}9J@B(0iհ bvP1{\P&G7D޴Iy_$-Qjm~Yrr&]CDv%bh|Yzni_ˆR;kg}nJOIIwyuL}{ЌNj}:+3Y?:WJ/N+Rzd=hb;dj͒suݔ@NKMԄ jqzC5@y°hL m;*5ezᕏ=ep XL n?מ:r`۵tŤZ|1v`V뽧_csج'ߤ%oTuumk%%%h)uy]Nk[n 'b2 l.=͜E%gf$[c;s:V-͞WߤWh-j7]4=F-X]>ZLSi[Y*We;Zan(ӇW|e(HNNP5[= r4tP &0<pc#`vTNV GFqvTi*Tyam$ߏWyE*VJKMTfFw>'$-ؽ.Ho.8c"@DADADADADADADADADA~j*֘,N;Pi3599h=goضLgiJ5փy~}&Zd9p֚ e:|hL``b/d9p? fgg+%%hMgXosج, ΩOl0Zh=xdjLmhݻoO[g_l,8a]٭+ӧ0$I]c]:粹:Teꢢ"5a^Kgh,&= =՟^߶“ߢE ܹS J}I%:8 IDAT~,9/ʃPW'Mo}zNƍ쨓zPbNZ~^z=4mswg;5 Y~SVMRXUյڱRf?s:w ;6H:ºi5-maM&O3;1IKeamZh͛7+##v+c ~u~ca]GnF'ټL~PPPbn voC4R,ӟgg %hq}@#M4IÇ Oy^xMZx ) yOw@HkN˖-Sǎmb]X@n+i͖!++K3gd\$mt$^YfJ\8PRF)77Wא!Cl$i:@@_oG I{$# 8磌ŋ91A (Im7֭>}ߴJq7ޗt^ -[ԩSj*}%]&' -ɓ'ꫯVzzvB#;a 7@GxI{j޼ƌ.LÇWBB7`O"I$/@R @eee@۷>}0,ɒ2$53Xs|cS~rpTYYY} kHc %&k.], @ADADADADADADADADA@lT<%''*Lo^={رc5h %$+CnܸQ3fҥK}vUVVs9G R,_{xˇ3o߾;TTTd}馛]uuuG~iԩ@4bnvmvfϞ /Peeeq}}za I~,誫{UWW뮻}_~YƍSMMMYχ֝waw\ďcxꩧtEƍկ_?۷5@u?1kNׯWzz/wy>}zj3 k(ٺuq_Zvf̘:~ ABQ&r|!%KҥKgԞ={<_X-z !CyFUUz~ ABQIIIjݺW$UXXDٳZ~ ABQƍecW$<(~<RSSvZujjjԧOZQu@4 8m&&&jԩg$ď1h ͟?_{768@g =@`)))5o6m3)ѣƌJ;wҿUTT /KZR{~a=@0o<*狔iFɶ[ˎ;T]]OX@?K.ۈxN pppppppppppppppppPfl߾] ,{ァk۶mڿo5BTӦMӴiӴ|r DB2e|An!Dy'tkΝ[A $***t5' "!駟oaDnΝ:t֭[gDШQ06qD;@ x M6v(PiizmZ4ew"@̴ixf [~-Fٱc&IZ2|n!?$@{[HTɏ#@hȎI# _m(F /6Z3z'\r,r!;w2Z3j=~GY7"I$iI.p_"?pN`y DD?: _  Gÿab7J !Bx@0 Bo cG@`1C[@0G @`0C_u V1 aCX>W ` | `!<S `"<. `#c`?cAC4 ?c p#~@0?:08&_MQ1J h#?/`7;I  q 7a wQ A 1 Hp !#<8/#@1Ul7=S=K.4Z?E_$i@!1!E4?`P_  @Bă10#: "aU,xbFY1 [n|n #'vEH:`xb #vD4Y hi.i&EΖv#O H4IŶ}:Ikh @tZRF#(tXҙzZ ?I3l7q@õ|ۍ1,GpuY Ꮿ@hJv#xxk$ v#9 5 }_$c S#=+"K{F*m7`#%H:NRSp6I?sIՖ{Ap$I$I:QRv2$Z @UJ*$]<FO4IENDB`