From 40046d98619b1974abd4f1369ac20002815118ef Mon Sep 17 00:00:00 2001 From: Constantine Date: Sun, 20 May 2018 10:43:36 +0300 Subject: [PATCH 01/66] Initial commit --- LICENSE | 674 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 674 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From 57394af4061e9a79483a8eccef3d406aa37566d4 Mon Sep 17 00:00:00 2001 From: Constantine Date: Sun, 20 May 2018 10:45:34 +0300 Subject: [PATCH 02/66] 2010 release for linux-2.6.36 --- LKM.svg | 8259 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 8259 insertions(+) create mode 100644 LKM.svg diff --git a/LKM.svg b/LKM.svg new file mode 100644 index 0000000..fccb0b4 --- /dev/null +++ b/LKM.svg @@ -0,0 +1,8259 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + Linux kernel map + + + Constantine Shulyupin + + + + + + © 2007, 2010 Constantine Shulyupin + + + www.MakeLinux.net/kernel_map + + + + + + + + + + + + + + + virtual + + + logical + + + + + + + + + + + electronics + + + + I/O + + + + + + memory + + + + CPU + + + HI char devices + + + HI subsystems + + + protocol families + + + sockets access + + + protocols + + + network interface + + + networking + + + Virtual File System + + + block devices + + + storage + + + virtual memory + + + memory access + + + logical memory + + + Page Allocator + + + memory + + + threads + + + processes + + + Scheduler + + + interrupts core + + + CPU specific + + + processing + + + generic HW access + + + system run + + + system + + + functionalities + + + layers + + + interfaces core + + + + Linux kernel map + + + logicalfile systems + abstract devicesand HID class drivers + HI peripheralsdevice drivers + + 2.6.36 + + networkdevice drivers + devicecontrol + + disk controllerdrivers + physical memoryoperations + device accessand bus drivers + user spaceinterfaces + hardwareinterfaces + files & directoriesaccess + user peripherals + disk controllers + network controllers + human interface + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + synchronization + + + Device Model + + + swap + + networkingstorage + memorymapping + + security + + + bridges + + + debugging + + + page cache + + socketsplice + + + + sys_init_module + + + timer_interrupt + + jiffies_64 + do_timer + tick_periodic + + context_switch + + alloc_file + + registers + + + RAM + + + MMU + + + I/O ports + + + I/O mem + + + keyboard + + + mouse + + + audio + + + graphics card + + + SCSI + + + SATA + + + DMA + + + Ethernet + + + WiFi + + + APIC + + + physically mapped memory + + + system files + + + copy_from_user + + + + © 2007, 2010 Constantine Shulyupin www.MakeLinux.net/kernel_map + + + + sys_write + + + sys_open + + + sys_execve + + + linux_binfmt + + + vfs_read + + + task_struct + + + usb_driver + + + sys_socketcall + + + socket + + + interrupt + + + sys_fork + + + schedule + + + do_IRQ + + + rq + + + kmalloc + + + kmem_cache + + + vmalloc + + + vmlist + + + page + + + do_page_fault + + + outw + + + cdev + + + /sysfs + + + /dev + + + readw + + + /proc + + + cdev_add + + + oss + + + mousedev + + + kbd + + + i8042_driver + + + psmouse + + + atkbd_drv + + + tty + + + console + + + snd_fops + + + video_fops + + + console_fops + + + vga_con + + + pt_regs + + + __get_free_pages + + + vm_struct + + + sys_mmap + + + /proc/self/maps + + + timer_list + + + do_softirq + + + tasklet_struct + + + request_queue + + + setup_irq + + + init_scsi + + + ext4_file_operations + + + gendisk + + + block_device_operations + + + sys_sync + + + sys_nanosleep + + + schedule_timeout + + + sysfs_ops + + + mm_struct + + + module + + + cdev_map + + + request_region + + + proto_ops + + + socket_file_ops + + + /proc/net/protocols + + + proto + + + tcp_prot + + + inet_stream_ops + + + inet_dgram_ops + + + udp_prot + + + inet_family_ops + + + __sock_create + + + ip_rcv + + + net_device + + + alloc_netdev_mq + + + ieee80211_alloc_hw + + + file_operations + + + sys_syslog + + + aic94xx_init + + + usb_hcd + + + ehci_irq + + + usb_hcd_irq + + + usb_submit_urb + + + ehci_urb_enqueue + + + usb_hcd_giveback_urb + + + pci_driver + + + start_kernel + + + init/main.c + + + run_init_process + + + do_initcalls + + + sys_reboot + + + do_mmap_pgoff + + + sys_brk + + + arch/x86/ + + + irq_desc + + + setup_timer + + + process_timeout + + + activate_task + + + sys_clone + + + sys_vfork + + + file + + + vm_area_struct + + + inode + + + fs/exec.c + + + address_space + + + ip_queue_xmit + + + dev_queue_xmit + + + netif_rx + + + ether_setup + + + ieee80211_xmit + + + ieee80211_rx + + + sd_fops + + + scsi_device + + + scsi_driver + + + sys_socket + + + linux/syscalls.h + + + linux/uaccess.h + + + pci_read + + + pci_write + + + ioremap + + + request_mem_region + + + kernel_power_off + + + kernel_restart + + + writew + + + inw + + + inet_create + + + vfs_write + + + ipw2100_pci_init_one + + + zd1201_probe + + + unix_family_ops + + + sys_mount + + + load_module + + + ext4_get_sb + + + file_system_type + + + get_sb + + + super_block + + + __alloc_pages + + + die + + + /proc/interrupts + + + cli + + + sti + + + switch_to + + + system_call + + + trap_init + + + sys_read + + + do_path_lookup + + + vfs_create + + + kernel/sched.c + + + drivers/net/ + + + show_regs + + + block/ + + + drivers/ + + + drivers/input/ + + + sound/ + + + drivers/media/ + + + init/ + + + kernel/ + + + include/asm/ + + + mm/slob.c + + + /proc/slabinfo + + + ac97_driver + + + usb_storage_driver + + + bus_type + + + device + + + device_driver + + + probe + + + class + + + device_create + + + driver_register + + + drivers/base/ + + + kobject + + + security/ + + + linux/security.h + + + selinux_ops + + + security_ops + + + security_socket_create + + + security_inode_create + + + pci_register_driver + + + ahci_pci_driver + + + libata + + + Scsi_Host + + + may_open + + + create_workqueue + + + alloc_skb + + + alsa + + + inode_operations + + + ramfs_fs_type + + + iscsi_tcp_transport + + + smb_fs_type + + + cifs_file_ops + + + nfs_file_operations + + + sk_buff + + + mm/mmap.c + + + vma_link + + + start_thread + + + find_vma_prepare + + + virt_to_page + + + fb_ops + + + pci_request_regions + + + fb_fops + + + cdev_add + + + register_chrdev + + + kset + + + msleep + + + do_fork + + + kernel_thread + + + current + + + thread_info + + + semaphore + + + workqueue_struct + + + work_struct + + + kthread_create + + + wake_up + + + atomic_t + + + mutex + + + add_timer + + + down_interruptible + + + kswapd + + + do_swap_page + + + fs/ + + + mm/ + + + kernel/ + + + net/ + + + kmem_cache_alloc + + + kernel/ + + + mousedev_handler + + + input_fops + + + get_page_from_freelist + + + wakeup_kswapd + + + try_to_free_pages + + + zone + + + drivers/media/video/ + + + video_device + + + NF_HOOK + + + nf_hooks + + + tcp_transmit_skb + + drivers,registers and interrupts + tcp_sendmsg + tcp_recvmsg + udp_sendmsg + udp_recvmsg + + netif_receive_skb + + + linux/netdevice.h + + ip_output + System Call Interface + + /dev/mem + + + mem_fops + + + mmap_mem + + + sock_ioctl + + + dev_ioctl + + linux/device.h + linux/kobject.h + device_type + driver_init + + arch/x86/mm/ + + sys_signal + ++ + request_irq + sys_times + sys_time + sys_gettimeofday + sys_futex + system callsand system files + cross-functionalmodules + sys_mprotect + sys_pivot_root + mount_root + kernel/signal.c + sys_kill + shm_vm_ops + sys_shmctl + sys_shmat + sys_newfstat + sys_select + sys_chroot + kvm_dev_ioctl + kvm + + camera + + interruptcontroller + USBcontroller + PCIcontroller + uvc_driver + sys_ioctl + register_netdev + lock_kernel + kernel_flag + do_sigaction + sys_pipe + sys_fsync + vfs_fsync + bdi_writeback_thread + do_writepages + fget + fd + sys_sysfs + file_systems + sys_flock + sys_mkdir + sys_inotify_init + sys_chdir + vfs_getattr + vfs_fstat + sys_epoll_create + kmem_cache_alloc + inode_permission + notify_change + inode_setattr + sys_chmod + sys_readv + iovec + sys_poll + sys_tee + sys_sysinfo + sys_swapon + swap_info + sys_msync + do_mmap + + up + + mutex_unlock + mutex_lock_interruptible + pgd_t + pmd_t + pte_t + setup_arch + mm_init + kmem_cache_init + vm_stat + sys_capset + x86_init + zonelist + kfree + __free_pages + __free_one_page + security_capset + + handle_sysrq + + + printk + + + log_buf + + + kgdb_breakpoint + + sys_ptrace + oprofile_init + oprofile_start + register_kprobe + kernel_param + native_init_IRQ + set_intr_gate + schedule_work + tasklet_action + softirq_init + module_param + bus_register + mem_init + vmalloc_init + /sys/class/ + kobject_uevent_init + kobject_uevent  + fsnotify + sys_fanotify_init + drm_driver + out_of_memory + vfree + . + . + sys_chown + fsnotify_change + fanotify_handle_event + /proc/meminfo + totalram_pages + num_physpages + + INIT_WORK + queue_work + usb_stor_host_template + scsi_host_alloc + sys_getdents + ext4_readdir + generic_file_aio_read + free_list + free_area + NR_FREE_PAGES + /proc/net/ + tcp4_seq_show + sg_proc_seq_show_dev + rt_cache_seq_show + sys_connect + sys_accept + sys_bind + sys_listen + sys_sendmsg + sys_recvmsg + sys_setsockopt + sock_sendpage + sock_splice_read + sys_sendfile + do_splice_direct + sys_splice + e1000_xmit_frame + e1000_intr + usbnet_probe + netif_carrier_on + ip_route_input + udp_rcv + tcp_v4_rcv + ip_local_deliver + ip_push_pending_frames + functionsimplementations + boot, shutdownpower management + hibernate + machine_ops + early_trap_init + inet_init + udp_sendpage + tcp_sendpage + tcp_splice_read + spin_unlock_irqrestore + spin_lock_irqsave + wait_event + wait_for_completion + + complete + owner + run_timer_softirq + si_meminfo + si_swapinfo + sys_mincore + + ACPI + + + mm/slub.c + + + mm/slab.c + + + From d568b533736c322fbd5ba96ac7172c7e71d92dd4 Mon Sep 17 00:00:00 2001 From: Constantine Date: Sun, 20 May 2018 11:33:02 +0300 Subject: [PATCH 03/66] Inkscape note --- README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..287e251 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# linux_kernel_map +Linux kernel map + +Edited with Inkscape 0.46 on Ubuntu 9.04. +Latter vesions of Inkscape don't render connetors propely. From 7cd42da2fe43d2afe8bc796407d0a9684627fc5b Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Wed, 18 Jul 2018 21:46:21 +0300 Subject: [PATCH 04/66] +srcxray.py --- srcxray.py | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100755 srcxray.py diff --git a/srcxray.py b/srcxray.py new file mode 100755 index 0000000..203d7b7 --- /dev/null +++ b/srcxray.py @@ -0,0 +1,172 @@ +#!/usr/bin/python3 +# +# srcxray - source code X-ray +# +# Analyzes interconnections between functions and structures in source code. +# +# Uses cscope and git grep --show-function to +# reveal references between identifiers. +# +# 2018 Constantine Shulyupin, const@MakeLinux.com +# + +import inspect, random, os, sys, collections, subprocess, re + +black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', + 'u32', + 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'trace_hardirqs_off'] + +level_limit = 7 +limit = 10000 +n = 0 + +def print_limited(a): + print(a) + global n + n +=1 + if n > limit: + print('Reached limit') + sys.exit(1) + +def log(*args, **kwargs): + print(inspect.stack()[1][3], str(*args).rstrip(), file=sys.stderr, **kwargs) + pass + +def popen(p): + return subprocess.Popen(p, shell=True, stdout=subprocess.PIPE, encoding="utf-8").stdout + +def extract_referer(line): + line = re.sub(r'__ro_after_init', '', line) + line = re.sub(r'FNAME\((\w+)\)', r'\1', line) + line = re.sub(r'.*TRACE_EVENT.*', '', line) + m = re.match(r'^[^\s]+=[^,]*\(\*(\b\w+)\)\s*[\(\[=][^;]*$', line) + if not m: + m = re.match(r'^[^\s]+=[^,]*(\b\w+)\s*[\(\[=][^;]*$', line) + if m: + return m.group(1) + +def extract_referer_test(): + for a in { + "fs=good2()", + "f=static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))", + "f=int good(a, bad (*func)(arg))", + "f=EXPORT_SYMBOL_GPL(bad);", + "f=bad (*good)()", + "f=int FNAME(good)(a)", + "f=TRACE_EVENT(a)", + "f: a=in bad()" }: + print(a, '->', extract_referer(a)) + +def func_referers_git_grep(name): + res = [] + r = None + for line in popen(r'git grep --no-index --word-regexp --show-function "^\s.*\b%s"'%(name)): + # Filter out names in comment afer function, when comment start from ' *' + # To see the problem try "git grep -p and" + if re.match(r'.*: \* ', line): + r = None + if r and r != name and not r in black_list: + res.append(r) + r = None + r = extract_referer(line) + return res + +cscope_warned = False + +def func_referers_cscope(name): + global cscope_warned + if not os.path.isfile('cscope.out'): + if not cscope_warned: + print("Recommended: cscope -bkR", file=sys.stderr) + cscope_warned = True + return [] + res = [ l.split()[1] for l in popen(r'cscope -d -L3 "%s"'%(name)) if not l in black_list] + if not res: + res = func_referers_git_grep(name) + return res + +def func_referers_all(name): + return set(func_referers_git_grep(name) + func_referers_cscope(name)) + +def referers_tree(name, referer = None, printed = None, level = 0): + if not referer: + if os.path.isfile('cscope.out'): + referer = func_referers_cscope + else: + print("Using git grep only, recommended to run: cscope -bkR", file=sys.stderr) + referer = func_referers_git_grep + if isinstance(referer, str): + referer = eval(referer) + if not printed: printed = set() + if name in printed: + print_limited(level*'\t' + name + ' ^') + return + else: + print_limited(level*'\t' + name) + printed.add(name) + if level > level_limit - 2: + print_limited((level + 1)*'\t' + '...') + return '' + listed = set() + for a in referer(name): + if a in listed: + continue + referers_tree(a, referer, printed, level + 1) + listed.add(a) + return '' + +def call_tree(node, printed = None, level = 0): + if not os.path.isfile('cscope.out'): + print("Please run: cscope -bkR", file=sys.stderr) + return False + if printed == None: printed = set() + if node in printed: + limit= - 1 + print_limited(level*'\t' + node + '^') + return + else: + print_limited(level*'\t' + node) + printed.add(node) + if level > level_limit - 2 : + print_limited((level + 1)*'\t' + '...') + return '' + local_printed = set() + for line in popen('cscope -d -L2 "%s"'%(node)): + I = line.split()[1]; + if I in local_printed or I in black_list: continue; + local_printed.add(I) + try: + call_tree(line.split()[1], printed, level + 1); + except: + pass + return '' + +me = os.path.basename(sys.argv[0]) + +def usage(): + print(me, "referers_tree", "") + print(me, "call_tree", "") + print("Try this:") + print("cd linux/init") + print(me, "referers_tree nfs_root_data") + print(me, "call_tree start_kernel") + print(me, "Emergency termination: ^Z, kill %1") + +def main(): + try: + ret = False + if len(sys.argv) == 1: + print('Run', me, 'usage') + else: + if '(' in sys.argv[1]: + ret = eval(sys.argv[1]) + else: + ret = eval(sys.argv[1] + '(' + ', '.join("'%s'"%(a) for a in sys.argv[2:]) + ')') + if type(ret) == type(False) and ret == False: + sys.exit(os.EX_CONFIG) + print(ret) + except KeyboardInterrupt: + warning("\nInterrupted") + +if __name__ == "__main__": + main() From a3104b157ece8ea63281393187aed6d85c8adb63 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Wed, 18 Jul 2018 23:48:22 +0300 Subject: [PATCH 05/66] fix spacing --- srcxray.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/srcxray.py b/srcxray.py index 203d7b7..2707e62 100755 --- a/srcxray.py +++ b/srcxray.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 # -# srcxray - source code X-ray +# srcxray - source code X-ray # # Analyzes interconnections between functions and structures in source code. # @@ -20,14 +20,16 @@ limit = 10000 n = 0 + def print_limited(a): print(a) global n - n +=1 + n += 1 if n > limit: print('Reached limit') sys.exit(1) + def log(*args, **kwargs): print(inspect.stack()[1][3], str(*args).rstrip(), file=sys.stderr, **kwargs) pass @@ -54,13 +56,13 @@ def extract_referer_test(): "f=bad (*good)()", "f=int FNAME(good)(a)", "f=TRACE_EVENT(a)", - "f: a=in bad()" }: + "f: a=in bad()"}: print(a, '->', extract_referer(a)) def func_referers_git_grep(name): res = [] r = None - for line in popen(r'git grep --no-index --word-regexp --show-function "^\s.*\b%s"'%(name)): + for line in popen(r'git grep --no-index --word-regexp --show-function "^\s.*\b%s"' % (name)): # Filter out names in comment afer function, when comment start from ' *' # To see the problem try "git grep -p and" if re.match(r'.*: \* ', line): @@ -80,7 +82,7 @@ def func_referers_cscope(name): print("Recommended: cscope -bkR", file=sys.stderr) cscope_warned = True return [] - res = [ l.split()[1] for l in popen(r'cscope -d -L3 "%s"'%(name)) if not l in black_list] + res = [l.split()[1] for l in popen(r'cscope -d -L3 "%s"'%(name)) if not l in black_list] if not res: res = func_referers_git_grep(name) return res @@ -88,7 +90,7 @@ def func_referers_cscope(name): def func_referers_all(name): return set(func_referers_git_grep(name) + func_referers_cscope(name)) -def referers_tree(name, referer = None, printed = None, level = 0): +def referers_tree(name, referer=None, printed = None, level = 0): if not referer: if os.path.isfile('cscope.out'): referer = func_referers_cscope @@ -122,17 +124,17 @@ def call_tree(node, printed = None, level = 0): if printed == None: printed = set() if node in printed: limit= - 1 - print_limited(level*'\t' + node + '^') + print_limited(level*'\t' + node + ' ^') return else: print_limited(level*'\t' + node) printed.add(node) - if level > level_limit - 2 : + if level > level_limit - 2: print_limited((level + 1)*'\t' + '...') return '' local_printed = set() for line in popen('cscope -d -L2 "%s"'%(node)): - I = line.split()[1]; + I = line.split()[1] if I in local_printed or I in black_list: continue; local_printed.add(I) try: @@ -158,7 +160,7 @@ def main(): if len(sys.argv) == 1: print('Run', me, 'usage') else: - if '(' in sys.argv[1]: + if '(' in sys.argv[1]: ret = eval(sys.argv[1]) else: ret = eval(sys.argv[1] + '(' + ', '.join("'%s'"%(a) for a in sys.argv[2:]) + ')') From 43dfadbc3c293130c27a777ddc4631bfd73159c2 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 07:52:33 +0300 Subject: [PATCH 06/66] pep8 --- srcxray.py | 61 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/srcxray.py b/srcxray.py index 2707e62..a0486b2 100755 --- a/srcxray.py +++ b/srcxray.py @@ -10,11 +10,16 @@ # 2018 Constantine Shulyupin, const@MakeLinux.com # -import inspect, random, os, sys, collections, subprocess, re +import inspect +import random +import os +import sys +import collections +import subprocess +import re -black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', - 'u32', - 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'trace_hardirqs_off'] +black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', 'u32', + 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'trace_hardirqs_off'] level_limit = 7 limit = 10000 @@ -31,12 +36,15 @@ def print_limited(a): def log(*args, **kwargs): - print(inspect.stack()[1][3], str(*args).rstrip(), file=sys.stderr, **kwargs) + print(inspect.stack()[1][3], + str(*args).rstrip(), file=sys.stderr, **kwargs) pass + def popen(p): return subprocess.Popen(p, shell=True, stdout=subprocess.PIPE, encoding="utf-8").stdout + def extract_referer(line): line = re.sub(r'__ro_after_init', '', line) line = re.sub(r'FNAME\((\w+)\)', r'\1', line) @@ -47,6 +55,7 @@ def extract_referer(line): if m: return m.group(1) + def extract_referer_test(): for a in { "fs=good2()", @@ -59,6 +68,7 @@ def extract_referer_test(): "f: a=in bad()"}: print(a, '->', extract_referer(a)) + def func_referers_git_grep(name): res = [] r = None @@ -73,8 +83,10 @@ def func_referers_git_grep(name): r = extract_referer(line) return res + cscope_warned = False + def func_referers_cscope(name): global cscope_warned if not os.path.isfile('cscope.out'): @@ -82,24 +94,29 @@ def func_referers_cscope(name): print("Recommended: cscope -bkR", file=sys.stderr) cscope_warned = True return [] - res = [l.split()[1] for l in popen(r'cscope -d -L3 "%s"'%(name)) if not l in black_list] + res = [l.split()[1] for l in popen(r'cscope -d -L3 "%s"' % + (name)) if not l in black_list] if not res: res = func_referers_git_grep(name) return res + def func_referers_all(name): return set(func_referers_git_grep(name) + func_referers_cscope(name)) -def referers_tree(name, referer=None, printed = None, level = 0): + +def referers_tree(name, referer=None, printed=None, level=0): if not referer: if os.path.isfile('cscope.out'): referer = func_referers_cscope else: - print("Using git grep only, recommended to run: cscope -bkR", file=sys.stderr) + print("Using git grep only, recommended to run: cscope -bkR", + file=sys.stderr) referer = func_referers_git_grep if isinstance(referer, str): referer = eval(referer) - if not printed: printed = set() + if not printed: + printed = set() if name in printed: print_limited(level*'\t' + name + ' ^') return @@ -117,13 +134,15 @@ def referers_tree(name, referer=None, printed = None, level = 0): listed.add(a) return '' -def call_tree(node, printed = None, level = 0): + +def call_tree(node, printed=None, level=0): if not os.path.isfile('cscope.out'): - print("Please run: cscope -bkR", file=sys.stderr) - return False - if printed == None: printed = set() + print("Please run: cscope -bkR", file=sys.stderr) + return False + if printed == None: + printed = set() if node in printed: - limit= - 1 + limit = - 1 print_limited(level*'\t' + node + ' ^') return else: @@ -133,18 +152,21 @@ def call_tree(node, printed = None, level = 0): print_limited((level + 1)*'\t' + '...') return '' local_printed = set() - for line in popen('cscope -d -L2 "%s"'%(node)): + for line in popen('cscope -d -L2 "%s"' % (node)): I = line.split()[1] - if I in local_printed or I in black_list: continue; + if I in local_printed or I in black_list: + continue local_printed.add(I) try: - call_tree(line.split()[1], printed, level + 1); + call_tree(line.split()[1], printed, level + 1) except: pass return '' + me = os.path.basename(sys.argv[0]) + def usage(): print(me, "referers_tree", "") print(me, "call_tree", "") @@ -154,6 +176,7 @@ def usage(): print(me, "call_tree start_kernel") print(me, "Emergency termination: ^Z, kill %1") + def main(): try: ret = False @@ -163,12 +186,14 @@ def main(): if '(' in sys.argv[1]: ret = eval(sys.argv[1]) else: - ret = eval(sys.argv[1] + '(' + ', '.join("'%s'"%(a) for a in sys.argv[2:]) + ')') + ret = eval(sys.argv[1] + '(' + ', '.join("'%s'" % (a) + for a in sys.argv[2:]) + ')') if type(ret) == type(False) and ret == False: sys.exit(os.EX_CONFIG) print(ret) except KeyboardInterrupt: warning("\nInterrupted") + if __name__ == "__main__": main() From ff0830b7ababd1f50cd086872f8a81bfcd59a5f2 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 08:29:29 +0300 Subject: [PATCH 07/66] pep8, pycodestyle --- srcxray.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/srcxray.py b/srcxray.py index a0486b2..35b9513 100755 --- a/srcxray.py +++ b/srcxray.py @@ -42,7 +42,8 @@ def log(*args, **kwargs): def popen(p): - return subprocess.Popen(p, shell=True, stdout=subprocess.PIPE, encoding="utf-8").stdout + return subprocess.Popen(p, shell=True, stdout=subprocess.PIPE, + encoding="utf-8").stdout def extract_referer(line): @@ -59,7 +60,8 @@ def extract_referer(line): def extract_referer_test(): for a in { "fs=good2()", - "f=static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))", + "f=static int fastop(struct x86_emulate_ctxt *ctxt, " + + "void (*fop)(struct fastop *))", "f=int good(a, bad (*func)(arg))", "f=EXPORT_SYMBOL_GPL(bad);", "f=bad (*good)()", @@ -72,12 +74,14 @@ def extract_referer_test(): def func_referers_git_grep(name): res = [] r = None - for line in popen(r'git grep --no-index --word-regexp --show-function "^\s.*\b%s"' % (name)): - # Filter out names in comment afer function, when comment start from ' *' + for line in popen(r'git grep --no-index --word-regexp --show-function ' + r'"^\s.*\b%s"' % (name)): + # Filter out names in comment afer function, + # when comment start from ' *' # To see the problem try "git grep -p and" if re.match(r'.*: \* ', line): r = None - if r and r != name and not r in black_list: + if r and r != name and r not in black_list: res.append(r) r = None r = extract_referer(line) @@ -95,7 +99,7 @@ def func_referers_cscope(name): cscope_warned = True return [] res = [l.split()[1] for l in popen(r'cscope -d -L3 "%s"' % - (name)) if not l in black_list] + (name)) if l not in black_list] if not res: res = func_referers_git_grep(name) return res @@ -139,7 +143,7 @@ def call_tree(node, printed=None, level=0): if not os.path.isfile('cscope.out'): print("Please run: cscope -bkR", file=sys.stderr) return False - if printed == None: + if printed is None: printed = set() if node in printed: limit = - 1 @@ -153,13 +157,13 @@ def call_tree(node, printed=None, level=0): return '' local_printed = set() for line in popen('cscope -d -L2 "%s"' % (node)): - I = line.split()[1] - if I in local_printed or I in black_list: + a = line.split()[1] + if a in local_printed or a in black_list: continue - local_printed.add(I) + local_printed.add(a) try: call_tree(line.split()[1], printed, level + 1) - except: + except Exception: pass return '' @@ -187,8 +191,8 @@ def main(): ret = eval(sys.argv[1]) else: ret = eval(sys.argv[1] + '(' + ', '.join("'%s'" % (a) - for a in sys.argv[2:]) + ')') - if type(ret) == type(False) and ret == False: + for a in sys.argv[2:]) + ')') + if isinstance(ret, bool) and ret is False: sys.exit(os.EX_CONFIG) print(ret) except KeyboardInterrupt: From c7dd8a39069a602ba276f545f26672be5ffa9c25 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 08:34:54 +0300 Subject: [PATCH 08/66] +filters --- srcxray.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/srcxray.py b/srcxray.py index 35b9513..1403161 100755 --- a/srcxray.py +++ b/srcxray.py @@ -19,7 +19,8 @@ import re black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', 'u32', - 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'trace_hardirqs_off'] + 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'if', + 'trace_hardirqs_off'] level_limit = 7 limit = 10000 @@ -79,8 +80,14 @@ def func_referers_git_grep(name): # Filter out names in comment afer function, # when comment start from ' *' # To see the problem try "git grep -p and" - if re.match(r'.*: \* ', line): - r = None + for p in { + r'.*: \* .*%s', + r'.*/\*.*%s', + r'.*//.*%s', + r'.*".*\b%s\b.*"'}: + if re.match(p % (name), line): + r = None + break if r and r != name and r not in black_list: res.append(r) r = None From 2a1185a5229049e3f2baa1abc4b75047a179d18b Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 11:41:45 +0300 Subject: [PATCH 09/66] -try --- srcxray.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/srcxray.py b/srcxray.py index 1403161..d0d10fd 100755 --- a/srcxray.py +++ b/srcxray.py @@ -168,10 +168,10 @@ def call_tree(node, printed=None, level=0): if a in local_printed or a in black_list: continue local_printed.add(a) - try: - call_tree(line.split()[1], printed, level + 1) - except Exception: - pass + # try: + call_tree(line.split()[1], printed, level + 1) + # except Exception: + # pass return '' From 114ff08598544bba569d5e969af5b14ba591e7f1 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 11:45:49 +0300 Subject: [PATCH 10/66] call_dep --- srcxray.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index d0d10fd..e168f9c 100755 --- a/srcxray.py +++ b/srcxray.py @@ -175,12 +175,42 @@ def call_tree(node, printed=None, level=0): return '' +def call_dep(node, printed=None, level=0): + if not os.path.isfile('cscope.out'): + print("Please run: cscope -bkR", file=sys.stderr) + return False + if printed is None: + printed = set() + if node in printed: + return + calls = set() + for a in [line.split()[1] for line in + popen('cscope -d -L2 "%s"' % (node))]: + if a in black_list: + continue + calls.add(a) + if calls: + if level < level_limit - 1: + printed.add(node) + print(node, end=': ') + for a in calls: + print(a, end=' ') + print() + for a in calls: + call_dep(a, printed, level + 1) + else: + pass + # TODO: print terminal + # print('...') + return '' + + me = os.path.basename(sys.argv[0]) def usage(): - print(me, "referers_tree", "") - print(me, "call_tree", "") + for c in ["referers_tree", "call_tree", "call_dep"]: + print(me, c, "") print("Try this:") print("cd linux/init") print(me, "referers_tree nfs_root_data") From 2f38ba92bbc6ef75a062d1693337eba9e4e7b6dd Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 12:54:11 +0300 Subject: [PATCH 11/66] use set --- srcxray.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/srcxray.py b/srcxray.py index e168f9c..0a038ee 100755 --- a/srcxray.py +++ b/srcxray.py @@ -73,7 +73,7 @@ def extract_referer_test(): def func_referers_git_grep(name): - res = [] + res = set() r = None for line in popen(r'git grep --no-index --word-regexp --show-function ' r'"^\s.*\b%s"' % (name)): @@ -89,7 +89,7 @@ def func_referers_git_grep(name): r = None break if r and r != name and r not in black_list: - res.append(r) + res.add(r) r = None r = extract_referer(line) return res @@ -105,8 +105,8 @@ def func_referers_cscope(name): print("Recommended: cscope -bkR", file=sys.stderr) cscope_warned = True return [] - res = [l.split()[1] for l in popen(r'cscope -d -L3 "%s"' % - (name)) if l not in black_list] + res = set([l.split()[1] for l in popen(r'cscope -d -L3 "%s"' % + (name)) if l not in black_list]) if not res: res = func_referers_git_grep(name) return res From 7a4b81f25f924d1a79c411e823375c3c77956b21 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 12:54:54 +0300 Subject: [PATCH 12/66] *filter --- srcxray.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/srcxray.py b/srcxray.py index 0a038ee..545ca4d 100755 --- a/srcxray.py +++ b/srcxray.py @@ -76,12 +76,13 @@ def func_referers_git_grep(name): res = set() r = None for line in popen(r'git grep --no-index --word-regexp --show-function ' - r'"^\s.*\b%s"' % (name)): + r'"^\s.*\b%s" ' + '**.\[hc\] **.cpp **.cc **.hh' % (name)): # Filter out names in comment afer function, # when comment start from ' *' # To see the problem try "git grep -p and" for p in { - r'.*: \* .*%s', + r'.*:\s+\* .*%s', r'.*/\*.*%s', r'.*//.*%s', r'.*".*\b%s\b.*"'}: @@ -139,8 +140,6 @@ def referers_tree(name, referer=None, printed=None, level=0): return '' listed = set() for a in referer(name): - if a in listed: - continue referers_tree(a, referer, printed, level + 1) listed.add(a) return '' From 655a11555791e6d04afb12923c3914ef74e12bd4 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 14:02:40 +0300 Subject: [PATCH 13/66] +check_output --- srcxray.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index 545ca4d..b2e6450 100755 --- a/srcxray.py +++ b/srcxray.py @@ -43,8 +43,7 @@ def log(*args, **kwargs): def popen(p): - return subprocess.Popen(p, shell=True, stdout=subprocess.PIPE, - encoding="utf-8").stdout + return [str(a) for a in subprocess.check_output(p, shell=True).splitlines()] def extract_referer(line): From 306cf68ae73972b0d4e2951f6f865197a9f3e356 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 19 Jul 2018 14:03:30 +0300 Subject: [PATCH 14/66] +referers_dep --- srcxray.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index b2e6450..3c49a5f 100755 --- a/srcxray.py +++ b/srcxray.py @@ -144,6 +144,38 @@ def referers_tree(name, referer=None, printed=None, level=0): return '' +def referers_dep(name, referer=None, printed=None, level=0): + if not referer: + if os.path.isfile('cscope.out'): + referer = func_referers_cscope + else: + print("Using git grep only, recommended to run: cscope -bkR", + file=sys.stderr) + referer = func_referers_git_grep + if isinstance(referer, str): + referer = eval(referer) + if not printed: + printed = set() + if name in printed: + return + if level > level_limit - 2: + return '' + referers = set(referer(name)) + if referers: + printed.add(name) + print(name, end=': ') + for a in referers: + print(a, end=' ') + print() + for a in referers: + referers_dep(a, referer, printed, level + 1) + else: + pass + # TODO: print terminal + # print('...') + return '' + + def call_tree(node, printed=None, level=0): if not os.path.isfile('cscope.out'): print("Please run: cscope -bkR", file=sys.stderr) @@ -207,7 +239,7 @@ def call_dep(node, printed=None, level=0): def usage(): - for c in ["referers_tree", "call_tree", "call_dep"]: + for c in ["referers_tree", "call_tree", "referers_dep", "call_dep"]: print(me, c, "") print("Try this:") print("cd linux/init") From 360895f9d813109801d0046e111dad812cb50c3e Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Fri, 20 Jul 2018 22:06:51 +0300 Subject: [PATCH 15/66] *level_limit --- srcxray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index 3c49a5f..9f10574 100755 --- a/srcxray.py +++ b/srcxray.py @@ -22,7 +22,7 @@ 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'if', 'trace_hardirqs_off'] -level_limit = 7 +level_limit = 8 limit = 10000 n = 0 From 9a90b1155a07b8df430ac430bf12d31ab5c07289 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 21 Jul 2018 23:01:11 +0300 Subject: [PATCH 16/66] fix limit --- srcxray.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index 9f10574..ee5b7cf 100755 --- a/srcxray.py +++ b/srcxray.py @@ -31,9 +31,10 @@ def print_limited(a): print(a) global n n += 1 - if n > limit: - print('Reached limit') + if n > limit + 1: + print('...') sys.exit(1) + # raise(Exception('Reached limit')) def log(*args, **kwargs): From 9451a0699d377f5323e31c528312cc5b112081ba Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 21 Jul 2018 23:01:48 +0300 Subject: [PATCH 17/66] *popen --- srcxray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index ee5b7cf..239e88a 100755 --- a/srcxray.py +++ b/srcxray.py @@ -44,7 +44,8 @@ def log(*args, **kwargs): def popen(p): - return [str(a) for a in subprocess.check_output(p, shell=True).splitlines()] + return [a.decode('utf-8') for a in subprocess.check_output(p, shell=True) + .splitlines()] def extract_referer(line): From f9fc791e42c9ee238499c49aa5eaded23deb437d Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 21 Jul 2018 23:05:18 +0300 Subject: [PATCH 18/66] small fixes --- srcxray.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/srcxray.py b/srcxray.py index 239e88a..a1f5753 100755 --- a/srcxray.py +++ b/srcxray.py @@ -78,7 +78,7 @@ def func_referers_git_grep(name): r = None for line in popen(r'git grep --no-index --word-regexp --show-function ' r'"^\s.*\b%s" ' - '**.\[hc\] **.cpp **.cc **.hh' % (name)): + r'**.\[hc\] **.cpp **.cc **.hh' % (name)): # Filter out names in comment afer function, # when comment start from ' *' # To see the problem try "git grep -p and" @@ -108,7 +108,7 @@ def func_referers_cscope(name): cscope_warned = True return [] res = set([l.split()[1] for l in popen(r'cscope -d -L3 "%s"' % - (name)) if l not in black_list]) + (name)) if l not in black_list]) if not res: res = func_referers_git_grep(name) return res @@ -185,7 +185,6 @@ def call_tree(node, printed=None, level=0): if printed is None: printed = set() if node in printed: - limit = - 1 print_limited(level*'\t' + node + ' ^') return else: From 0a83ba373e25b4f641c3c4279e4d6cd1045a48e5 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 22 Jul 2018 00:32:07 +0300 Subject: [PATCH 19/66] + networkx,DiGraph, syscalls --- srcxray.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/srcxray.py b/srcxray.py index a1f5753..e64f2cf 100755 --- a/srcxray.py +++ b/srcxray.py @@ -17,6 +17,7 @@ import collections import subprocess import re +import networkx as nx black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', 'u32', 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'if', @@ -236,6 +237,84 @@ def call_dep(node, printed=None, level=0): return '' +def my_graph(name=None): + g = nx.DiGraph(name=name) + g.graph.update({'node': {'shape': 'none', 'fontsize': 50}}) + g.graph.update({'rankdir': 'LR', 'nodesep': 0, }) + return g + + +def syscalls(): + sc = my_graph('syscalls') + scd = 'SYSCALL_DEFINE.list' + if not os.path.isfile(scd): + os.system("grep SYSCALL_DEFINE -r --include='*.c' > " + scd) + with open(scd, 'r') as f: + v = set() + for s in f: + m = re.match(r'(.*?):.*SYSCALL.*\(([\w]+)', s) + if m: + for p in { + '^old', + '^xnew', + r'.*64', + r'.*32$', + r'.*16$', + }: + if re.match(p, m.group(2)): + m = None + break + if m: + syscall = m.group(2) + syscall = re.sub('^new', '', syscall) + if 'compat' in m.group(1): + continue + if syscall in v or 'compat' in m.group(1): + continue + v.add(syscall) + path = m.group(1).split('/') + if (m.group(1).startswith('arch/') + and not m.group(1).startswith('arch/x86')): + continue + p2 = '/'.join(path[1:]) + sc.add_edge(path[0] + '/', p2) + sc.add_edge(p2, syscall) + return sc + + +def digraph_print(dg): + def digraph_print_sub(node=None, printed=None, level=0): + outs = [_ for _ in dg.successors(node)] + if node in printed: + print_limited(level*'\t' + node + ' ^') + return + else: + s = ' ...' if level > level_limit - 2 and outs else '' + print_limited(level*'\t' + node + s) + printed.add(node) + if level > level_limit - 2: + return '' + passed = set() + for o in outs: + if o in passed or o in black_list: + continue + passed.add(o) + digraph_print_sub(o, printed, level + 1) + + starts = {} + printed = set() + for i in [n for (n, d) in dg.in_degree if not d]: + starts[i] = dg.out_degree(i) + starts = sorted(starts.items(), key=lambda k: k[1], reverse=True) + outs = [a[0] for a in starts] + passed = set() + for o in outs: + if o in passed or o in black_list: + continue + passed.add(o) + digraph_print_sub(o, printed) + + me = os.path.basename(sys.argv[0]) From 83f5dc482de21baaf198711933d06272faa9ba4b Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 22 Jul 2018 16:19:45 +0300 Subject: [PATCH 20/66] write_dot --- srcxray.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/srcxray.py b/srcxray.py index e64f2cf..188c524 100755 --- a/srcxray.py +++ b/srcxray.py @@ -18,6 +18,7 @@ import subprocess import re import networkx as nx +from networkx.drawing.nx_agraph import write_dot black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', 'u32', 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'if', @@ -282,6 +283,12 @@ def syscalls(): return sc +# DiGraph +# write_dot to_agraph AGraph +# agwrite +# srcxray.py 'write_dot(syscalls(), "syscalls.dot")' + + def digraph_print(dg): def digraph_print_sub(node=None, printed=None, level=0): outs = [_ for _ in dg.successors(node)] From 00b50926e3c355b37df0c5e25ae33ef30ee11fcc Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 22 Jul 2018 16:33:13 +0300 Subject: [PATCH 21/66] ret is not None --- srcxray.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index 188c524..d459792 100755 --- a/srcxray.py +++ b/srcxray.py @@ -322,6 +322,61 @@ def digraph_print_sub(node=None, printed=None, level=0): digraph_print_sub(o, printed) +def cflow_preprocess(a): + with open(a, 'r') as f: + for s in f: + # treat struct like function + s = re.sub(r"^static const struct (.*)\[\] = ", r"\1()", s) + s = re.sub(r"^static __initdata int \(\*actions\[\]\)\(void\) = ", + "int actions()", s) # treat struct like function + s = re.sub(r"^static ", "", s) + s = re.sub(r"COMPAT_SYSCALL_DEFINE[0-9]\((\w*),", + r"compat_sys_\1(", s) + s = re.sub(r"SYSCALL_DEFINE[0-9]\((\w*),", r"sys_\1(", s) + s = re.sub(r"__setup\(.*,(.*)\)", r"void __setup() {\1();}", s) + s = re.sub(r"early_param\(.*,(.*)\)", + r"void early_param() {\1();}", s) + s = re.sub(r"rootfs_initcall\((.*)\)", + r"void rootfs_initcall() {\1();}", s) + s = re.sub(r"^static ", "", s) + s = re.sub(r"__read_mostly", "", s) + s = re.sub(r"^inline ", "", s) + s = re.sub(r"^const ", "", s) + s = re.sub(r"^struct (.*) =", r"\1()", s) + s = re.sub(r"^struct ", "", s) + # for line in sys.stdin: + sys.stdout.write(s) + + +def import_cflow(): + cf = nx.DiGraph() + stack = list() + nprev = -1 + # "--depth=%d " %(level_limit+1) + + cflow = (r"cflow " + + "--preprocess='srcxray.py cflow_preprocess' " + + "--include=_sxt --brief --level-indent='0=\t' " + + " *.[ch] *.cpp *.hh ") + # " $(find -name '*.[ch]' -o -name '*.cpp' -o -name '*.hh') " + for line in popen(cflow): + # --print-level + m = re.match(r'^([\t]*)([^(^ ^<]+)', str(line)) + if m: + n = len(m.group(1)) + id = str(m.group(2)) + else: + raise Exception(line) + + if n <= nprev: + stack = stack[:n - nprev - 1] + # print(n, id, stack) + if len(stack): + cf.add_edge(stack[-1], id) + stack.append(id) + nprev = n + return cf + + me = os.path.basename(sys.argv[0]) @@ -348,7 +403,8 @@ def main(): for a in sys.argv[2:]) + ')') if isinstance(ret, bool) and ret is False: sys.exit(os.EX_CONFIG) - print(ret) + if (ret is not None): + print(ret) except KeyboardInterrupt: warning("\nInterrupted") From 82f32b248b3d21b4017f36c09a7f5e661a3e8fd3 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 22 Jul 2018 17:16:40 +0300 Subject: [PATCH 22/66] reduce_graph --- srcxray.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index d459792..34597a4 100755 --- a/srcxray.py +++ b/srcxray.py @@ -245,6 +245,17 @@ def my_graph(name=None): return g +def reduce_graph(g): + print(type(g)) + rm = set() + for e in g: + if not g.out_degree(e): + rm.add(e) + print(rm) + g.remove_nodes_from(rm) + return g + + def syscalls(): sc = my_graph('syscalls') scd = 'SYSCALL_DEFINE.list' @@ -293,11 +304,11 @@ def digraph_print(dg): def digraph_print_sub(node=None, printed=None, level=0): outs = [_ for _ in dg.successors(node)] if node in printed: - print_limited(level*'\t' + node + ' ^') + print_limited(level*'\t' + str(node) + ' ^') return else: s = ' ...' if level > level_limit - 2 and outs else '' - print_limited(level*'\t' + node + s) + print_limited(level*'\t' + str(node) + s) printed.add(node) if level > level_limit - 2: return '' From cf871e501c47bd8cf2933b55b3686529b5a5dc67 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:28:37 +0300 Subject: [PATCH 23/66] syscalls: +includes --- srcxray.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/srcxray.py b/srcxray.py index 34597a4..4f61487 100755 --- a/srcxray.py +++ b/srcxray.py @@ -256,8 +256,56 @@ def reduce_graph(g): return g +def includes(a): + res = [] + # log(a) + for a in popen('man -s 2 %s 2> /dev/null |' + ' head -n 20 | grep include || true' % (a), + shell=True): + m = re.match('.*<(.*)>', a) + if m: + res.append(m.group(1)) + if not res: + for a in popen('grep -l -r " %s *(" ' + '/usr/include --include "*.h" ' + '2> /dev/null || true' % (a)): + # log(a) + a = re.sub(r'.*/(bits)', r'\1', a) + a = re.sub(r'.*/(sys)', r'\1', a) + a = re.sub(r'/usr/include/(.*)', r'\1', a) + # log(a) + res.append(a) + res = set(res) + if res and len(res) > 1: + r = set() + for f in res: + # log('grep " %s \+\(" --include "%s" -r /usr/include/'%(a,f)) + # log(os.system( + # 'grep -w "%s" --include "%s" -r /usr/include/'%(a,f))) + if 0 != os.system( + 'grep " %s *(" --include "%s" -r /usr/include/ -q' + % (a, os.path.basename(f))): + r.add(f) + res = res.difference(r) + log(res) + return ','.join(list(res)) if res else 'unexported' + + def syscalls(): sc = my_graph('syscalls') + inc = 'includes.list' + if not os.path.isfile(inc): + os.system('ctags --langmap=c:+.h --c-kinds=+pex -I __THROW ' + + ' -R -u -f- /usr/include/ | cut -f1,2 > ' + + inc) + ''' + if False: + includes = {} + with open(inc, 'r') as f: + for s in f: + includes[s.split()[0]] = s.split()[1] + log(includes) + ''' scd = 'SYSCALL_DEFINE.list' if not os.path.isfile(scd): os.system("grep SYSCALL_DEFINE -r --include='*.c' > " + scd) From dfc4c0fb3ddbdab9b8d0d81e736fadfa229472fe Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:31:32 +0300 Subject: [PATCH 24/66] *syscalls --- srcxray.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/srcxray.py b/srcxray.py index 4f61487..39f81e6 100755 --- a/srcxray.py +++ b/srcxray.py @@ -310,8 +310,10 @@ def syscalls(): if not os.path.isfile(scd): os.system("grep SYSCALL_DEFINE -r --include='*.c' > " + scd) with open(scd, 'r') as f: - v = set() + v = set('sigsuspend', 'llseek', 'sysfs', 'sync_file_range2', 'ustat', 'bdflush') for s in f: + if any(x in s.lower() for x in ['compat', 'stub']): + continue m = re.match(r'(.*?):.*SYSCALL.*\(([\w]+)', s) if m: for p in { @@ -327,18 +329,30 @@ def syscalls(): if m: syscall = m.group(2) syscall = re.sub('^new', '', syscall) - if 'compat' in m.group(1): - continue - if syscall in v or 'compat' in m.group(1): - continue - v.add(syscall) path = m.group(1).split('/') - if (m.group(1).startswith('arch/') + if (m.group(1).startswith('mm/nommu.c') + or m.group(1).startswith('arch/x86/ia32') + or m.group(1).startswith('arch/') + or syscall.startswith('vm86') and not m.group(1).startswith('arch/x86')): continue + if syscall in v: + continue + v.add(syscall) p2 = '/'.join(path[1:]) - sc.add_edge(path[0] + '/', p2) - sc.add_edge(p2, syscall) + p2 = m.group(1) + # if log(difflib.get_close_matches(syscall,v) or ''): + # log(syscall) + # log(syscall + ' ' + (includes.get(syscall) or '------')) + # man -s 2 timerfd_settime | head -n 20 + # sc.add_edge('syscalls', path[0] + '/') + # sc.add_edge(path[0] + '/', p2) + # sc.add_edge(p2, syscall) + i = includes(syscall) + log(p2 + ' ' + str(i) + ' ' + syscall) + sc.add_edge(i, i+' - '+p2) + sc.add_edge(i+' - '+p2, syscall) + # sc.add_edge(includes(syscall), syscall) return sc From 3cd5da18fba2733362162654412b057791de1a94 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:32:47 +0300 Subject: [PATCH 25/66] *digraph_print --- srcxray.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/srcxray.py b/srcxray.py index 39f81e6..4d0d1df 100755 --- a/srcxray.py +++ b/srcxray.py @@ -375,20 +375,24 @@ def digraph_print_sub(node=None, printed=None, level=0): if level > level_limit - 2: return '' passed = set() - for o in outs: + for o in outs.keys(): if o in passed or o in black_list: continue passed.add(o) digraph_print_sub(o, printed, level + 1) - starts = {} printed = set() - for i in [n for (n, d) in dg.in_degree if not d]: - starts[i] = dg.out_degree(i) - starts = sorted(starts.items(), key=lambda k: k[1], reverse=True) - outs = [a[0] for a in starts] + if not starts: + starts = {} + for i in [n for (n, d) in dg.in_degree if not d]: + starts[i] = dg.out_degree(i) + starts = [a[0] for a in sorted(starts.items(), key=lambda k: k[1], reverse=True)] + if len(starts) > 1: + print_limited('starts') + for s in starts: + print_limited('\t' + s + ' ->') passed = set() - for o in outs: + for o in starts: if o in passed or o in black_list: continue passed.add(o) From e8a73d770bf00973e847deed7a39a03129ac167d Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:34:40 +0300 Subject: [PATCH 26/66] +most_used, + starts, *digraph_print --- srcxray.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index 4d0d1df..8d08337 100755 --- a/srcxray.py +++ b/srcxray.py @@ -362,9 +362,22 @@ def syscalls(): # srcxray.py 'write_dot(syscalls(), "syscalls.dot")' -def digraph_print(dg): + +def most_used(dg, ins=10, outs=10): + # return {a: b for a, b in sorted(dg.in_degree, key=lambda k: k[1]) if b > 1 and} + return [(x, dg.in_degree(x), dg.out_degree(x)) for x in dg.nodes() + if dg.in_degree(x) > ins and dg.out_degree(x) > outs] + + +def starts(dg): # roots + return {n: dg.out_degree(n) for (n, d) in dg.in_degree if not d} + + +def digraph_print(dg, starts=None, sort=False): def digraph_print_sub(node=None, printed=None, level=0): - outs = [_ for _ in dg.successors(node)] + outs = {_: dg.out_degree(_) for _ in dg.successors(node)} + if sort: + outs = {a: b for a, b in sorted(outs.items(), key=lambda k: k[1], reverse=True)} if node in printed: print_limited(level*'\t' + str(node) + ' ^') return From ec2bf8b481b22aa1a35e3171cc2665c473456d2d Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:35:40 +0300 Subject: [PATCH 27/66] +imports --- srcxray.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index 8d08337..256a589 100755 --- a/srcxray.py +++ b/srcxray.py @@ -18,7 +18,10 @@ import subprocess import re import networkx as nx -from networkx.drawing.nx_agraph import write_dot +from networkx.drawing.nx_agraph import * +from networkx.generators.ego import * +from pprint import pprint +import difflib black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', 'u32', 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'if', From f673fa83195efb57df2ff6d7f85722298f61a796 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:36:01 +0300 Subject: [PATCH 28/66] updates --- srcxray.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/srcxray.py b/srcxray.py index 256a589..8ef8c1f 100755 --- a/srcxray.py +++ b/srcxray.py @@ -28,7 +28,7 @@ 'trace_hardirqs_off'] level_limit = 8 -limit = 10000 +limit = 100000 n = 0 @@ -43,14 +43,14 @@ def print_limited(a): def log(*args, **kwargs): + s = str(*args).rstrip() print(inspect.stack()[1][3], - str(*args).rstrip(), file=sys.stderr, **kwargs) - pass + s, file=sys.stderr, **kwargs) + return s def popen(p): - return [a.decode('utf-8') for a in subprocess.check_output(p, shell=True) - .splitlines()] + return subprocess.check_output(p, shell=True).decode('utf-8').splitlines() def extract_referer(line): @@ -243,18 +243,16 @@ def call_dep(node, printed=None, level=0): def my_graph(name=None): g = nx.DiGraph(name=name) - g.graph.update({'node': {'shape': 'none', 'fontsize': 50}}) - g.graph.update({'rankdir': 'LR', 'nodesep': 0, }) + # g.graph.update({'node': {'shape': 'none', 'fontsize': 50}}) + # g.graph.update({'rankdir': 'LR', 'nodesep': 0, }) return g def reduce_graph(g): - print(type(g)) rm = set() for e in g: if not g.out_degree(e): rm.add(e) - print(rm) g.remove_nodes_from(rm) return g @@ -499,7 +497,7 @@ def main(): if (ret is not None): print(ret) except KeyboardInterrupt: - warning("\nInterrupted") + log("\nInterrupted") if __name__ == "__main__": From d895a55c5679a9caf215a84b972b23e59c255db6 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:36:16 +0300 Subject: [PATCH 29/66] +leaves --- srcxray.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index 8ef8c1f..dd5f8f7 100755 --- a/srcxray.py +++ b/srcxray.py @@ -361,7 +361,17 @@ def syscalls(): # write_dot to_agraph AGraph # agwrite # srcxray.py 'write_dot(syscalls(), "syscalls.dot")' - +# srcxray.py "write_dot(import_cflow(),'a.dot')" +# write_graphml +# F=sys_mount; srcxray.py "digraph_print(import_cflow(),['$F'])" > $F.tree +# srcxray.py "leaves(read_dot('a.dot'))" +# srcxray.py "most_used(read_dot('a.dot'))" +# srcxray.py "digraph_print(reduce_graph(reduce_graph(read_dot('a.dot'))))" +# srcxray.py "pprint(most_used(read_dot('a.dot')))" + +def leaves(dg): + # [x for x in G.nodes() if G.out_degree(x)==0 and G.in_degree(x)==1] + return {n: dg.in_degree(n) for (n, d) in dg.out_degree if not d} def most_used(dg, ins=10, outs=10): From 9dc8cf0cc6fe4f56e5739b97c138d587ebe641d1 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:36:33 +0300 Subject: [PATCH 30/66] *cflow_preprocess --- srcxray.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/srcxray.py b/srcxray.py index dd5f8f7..785a2ad 100755 --- a/srcxray.py +++ b/srcxray.py @@ -427,10 +427,13 @@ def cflow_preprocess(a): with open(a, 'r') as f: for s in f: # treat struct like function + s = re.sub(r"^static struct (.*) = ", r"\1()", s) + s = re.sub(r"^static struct (.*)\[\] = ", r"\1()", s) s = re.sub(r"^static const struct (.*)\[\] = ", r"\1()", s) s = re.sub(r"^static __initdata int \(\*actions\[\]\)\(void\) = ", "int actions()", s) # treat struct like function s = re.sub(r"^static ", "", s) + s = re.sub(r"SENSOR_DEVICE_ATTR.*\((\w*),", r"void sensor_dev_attr_\1()(", s) s = re.sub(r"COMPAT_SYSCALL_DEFINE[0-9]\((\w*),", r"compat_sys_\1(", s) s = re.sub(r"SYSCALL_DEFINE[0-9]\((\w*),", r"sys_\1(", s) From a515d61f0fb49f988a85840c59e32177e91a9418 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Thu, 26 Jul 2018 08:36:48 +0300 Subject: [PATCH 31/66] *import_cflow --- srcxray.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/srcxray.py b/srcxray.py index 785a2ad..05e4978 100755 --- a/srcxray.py +++ b/srcxray.py @@ -452,16 +452,25 @@ def cflow_preprocess(a): sys.stdout.write(s) -def import_cflow(): - cf = nx.DiGraph() +def import_cflow(a=None): + if not a: + # arg = "$(find -name '*.[ch]' -o -name '*.cpp' -o -name '*.hh')" + arg = "*.c *.h *.cpp *.hh " + elif isinstance(a, list): + pass + elif os.path.isdir(a): + pass + elif os.path.isfile(a): + arg = a + pass + cf = my_graph() stack = list() nprev = -1 # "--depth=%d " %(level_limit+1) + cflow = (r"cflow " + "--preprocess='srcxray.py cflow_preprocess' " + "--include=_sxt --brief --level-indent='0=\t' " + - " *.[ch] *.cpp *.hh ") - # " $(find -name '*.[ch]' -o -name '*.cpp' -o -name '*.hh') " + arg) for line in popen(cflow): # --print-level m = re.match(r'^([\t]*)([^(^ ^<]+)', str(line)) From aff84ae738f45178505134e781039d7b618065f4 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 28 Jul 2018 23:02:03 +0300 Subject: [PATCH 32/66] +cleanup *blacllist --- srcxray.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/srcxray.py b/srcxray.py index 05e4978..0bd547f 100755 --- a/srcxray.py +++ b/srcxray.py @@ -22,10 +22,29 @@ from networkx.generators.ego import * from pprint import pprint import difflib +import glob +from pathlib import * + +black_list = ('aligned __attribute__ unlikely typeof u32' + 'PVOP_CALLEE0 PVOP_VCALLEE0 PVOP_VCALLEE1 if trace_hardirqs_off' + 'i NULL likely unlikely true false test_bit NAPI_GRO_CB clear_bit ' + 'atomic_read preempt_disable preempt_enable container_of ENOSYS ' + 'READ_ONCE u64 u8 _RET_IP_ ret current ' + 'AT_FDCWD fdput EBADF file_inode ' + 'ssize_t path_put __user ' + 'list_empty memcpy size_t loff_t pos d_inode dput copy_to_user EIO bool out IS_ERR ' + 'EPERM rcu_read_lock rcu_read_unlock spin_lock spin_unlock list_for_each_entry kfree ' + 'GFP_KERNEL ENOMEM EFAULT ENOENT EAGAIN PTR_ERR PAGE_SHIFT PAGE_SIZE ' + 'pgoff_t pte_t pmd_t HPAGE_PMD_NR PageLocked entry swp_entry_t next unlock_page spinlock_t end start ' + ' VM_BUG_ON VM_BUG_ON_PAGE BDI_SHOW max ' + 'ssize_t path_put __user ' + 'list_del compound_head list_add cond_resched put_page nr_pages min spin_lock_irqsave IS_ENABLED ' + 'EBUSY UL NODE_DATA pr_err memset list size ptl PAGE_MASK pr_info offset addr get_page sprintf ' + 'INIT_LIST_HEAD NUMA_NO_NODE spin_unlock_irqrestore mutex_unlock mutex_lock ' + 'page_to_nid page_to_pfn pfn page_zone pfn_to_page' + 'BUG BUG_ON flags WARN_ON_ONCE ENODEV cpu_to_le16 cpumask_bits ' + 'ERR_PTR ENOTSUPP EOPNOTSUPP EOPNOTSUPP WARN_ON EINVAL ').split() -black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', 'u32', - 'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'if', - 'trace_hardirqs_off'] level_limit = 8 limit = 100000 @@ -366,9 +385,18 @@ def syscalls(): # F=sys_mount; srcxray.py "digraph_print(import_cflow(),['$F'])" > $F.tree # srcxray.py "leaves(read_dot('a.dot'))" # srcxray.py "most_used(read_dot('a.dot'))" +# srcxray.py "digraph_print(read_dot('a.dot'))" # srcxray.py "digraph_print(reduce_graph(reduce_graph(read_dot('a.dot'))))" # srcxray.py "pprint(most_used(read_dot('a.dot')))" +def cleanup(a): + dg = read_dot(a) + print(dg.number_of_edges()) + dg.remove_nodes_from(black_list) + print(dg.number_of_edges()) + write_dot(dg, a) + + def leaves(dg): # [x for x in G.nodes() if G.out_degree(x)==0 and G.in_degree(x)==1] return {n: dg.in_degree(n) for (n, d) in dg.out_degree if not d} @@ -483,8 +511,9 @@ def import_cflow(a=None): if n <= nprev: stack = stack[:n - nprev - 1] # print(n, id, stack) - if len(stack): - cf.add_edge(stack[-1], id) + if id not in black_list: + if len(stack): + cf.add_edge(stack[-1], id) stack.append(id) nprev = n return cf From dad62d5430fc79594410c57c63e1a78bd98356fc Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 28 Jul 2018 23:03:44 +0300 Subject: [PATCH 33/66] *import_cflow --- srcxray.py | 55 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/srcxray.py b/srcxray.py index 0bd547f..e808e30 100755 --- a/srcxray.py +++ b/srcxray.py @@ -404,8 +404,10 @@ def leaves(dg): def most_used(dg, ins=10, outs=10): # return {a: b for a, b in sorted(dg.in_degree, key=lambda k: k[1]) if b > 1 and} - return [(x, dg.in_degree(x), dg.out_degree(x)) for x in dg.nodes() - if dg.in_degree(x) > ins and dg.out_degree(x) > outs] + # return [(x, dg.in_degree(x), dg.out_degree(x)) + return [x + for x in dg.nodes() + if dg.in_degree(x) >= ins and dg.out_degree(x) >= outs] def starts(dg): # roots @@ -466,40 +468,63 @@ def cflow_preprocess(a): r"compat_sys_\1(", s) s = re.sub(r"SYSCALL_DEFINE[0-9]\((\w*),", r"sys_\1(", s) s = re.sub(r"__setup\(.*,(.*)\)", r"void __setup() {\1();}", s) - s = re.sub(r"early_param\(.*,(.*)\)", - r"void early_param() {\1();}", s) - s = re.sub(r"rootfs_initcall\((.*)\)", - r"void rootfs_initcall() {\1();}", s) + s = re.sub(r"^(\w*)param\(.*,(.*)\)", r"void \1param() {\2();}", s) + s = re.sub(r"(\w*)initcall\((.*)\)", + r"void \1initcall() {\2();}", s) s = re.sub(r"^static ", "", s) - s = re.sub(r"__read_mostly", "", s) + # s = re.sub(r"__read_mostly", "", s) s = re.sub(r"^inline ", "", s) s = re.sub(r"^const ", "", s) s = re.sub(r"^struct (.*) =", r"\1()", s) s = re.sub(r"^struct ", "", s) + # __attribute__ # for line in sys.stdin: sys.stdout.write(s) -def import_cflow(a=None): +cflow_param = { + "modifier": "__init __inline__ noinline __initdata __randomize_layout __read_mostly asmlinkage " + " __visible __init __leaf__ __ref", + "wrapper": "__attribute__ __section__ " + "TRACE_EVENT MODULE_AUTHOR MODULE_DESCRIPTION MODULE_LICENSE MODULE_LICENSE MODULE_SOFTDEP " + "__acquires __releases __ATTR" + # "wrapper": "__setup early_param" + } + +# export CPATH=:include:arch/x86/include:../build/include/:../build/arch/x86/include/generated/:include/uapi +# srcxray.py "'\n'.join(cflow('init/main.c'))" + + +def cflow(a): + arg = a if not a: # arg = "$(find -name '*.[ch]' -o -name '*.cpp' -o -name '*.hh')" arg = "*.c *.h *.cpp *.hh " + arg = " $(cat cscope.files)" elif isinstance(a, list): pass elif os.path.isdir(a): pass elif os.path.isfile(a): - arg = a pass + # "--depth=%d " %(level_limit+1) + + # --debug=1 + cflow = (r"cflow -v " + # + "-DCONFIG_KALLSYMSZ " + + "--preprocess='srcxray.py cflow_preprocess' " + + ''.join([''.join(["--symbol={0}:{1} ".format(w, p) + for w in cflow_param[p].split()]) + for p in cflow_param.keys()]) + + " --include=_sxt --brief --level-indent='0=\t' " + + a) + return popen(cflow) + + +def import_cflow(a=None): cf = my_graph() stack = list() nprev = -1 - # "--depth=%d " %(level_limit+1) + - cflow = (r"cflow " + - "--preprocess='srcxray.py cflow_preprocess' " + - "--include=_sxt --brief --level-indent='0=\t' " + - arg) - for line in popen(cflow): + for line in cflow(a): # --print-level m = re.match(r'^([\t]*)([^(^ ^<]+)', str(line)) if m: From fbf692a988e2e698e34f5b52fe0f98cb3743218e Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 28 Jul 2018 23:16:48 +0300 Subject: [PATCH 34/66] +cflow_linux --- srcxray.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/srcxray.py b/srcxray.py index e808e30..d00e0d1 100755 --- a/srcxray.py +++ b/srcxray.py @@ -544,6 +544,24 @@ def import_cflow(a=None): return cf +def cflow_linux(): + dirs = 'init fs ipc net fs/ext4 net lib block security crypto mm arch/x86/kernel '.split() + all = nx.DiGraph() + for a in dirs: + for c in glob.glob(os.path.join(a, "*.c")): + dot = str(Path(c).with_suffix(".dot")) + if not os.path.isfile(dot): + g = import_cflow(c) + write_dot(g, dot) + print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) + else: + g = read_dot(dot) + print(dot) + all.add_nodes_from(g.nodes()) + all.add_edges_from(g.edges()) + write_dot(all, 'all.dot') + + me = os.path.basename(sys.argv[0]) From 7b73c377c262a732a570c16b30c7d17b0dcaf261 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 00:26:33 +0300 Subject: [PATCH 35/66] +digraph_tree --- srcxray.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/srcxray.py b/srcxray.py index d00e0d1..49d1b27 100755 --- a/srcxray.py +++ b/srcxray.py @@ -414,6 +414,31 @@ def starts(dg): # roots return {n: dg.out_degree(n) for (n, d) in dg.in_degree if not d} +def digraph_tree(dg, starts=None): + tree = nx.DiGraph() + def sub(node): + tree.add_node(node) + for o in dg.successors(node): + if o in black_list or tree.has_edge(node, o): + continue + tree.add_edge(node,o) + sub(o) + + printed = set() + if not starts: + starts = {} + for i in [n for (n, d) in dg.in_degree if not d]: + starts[i] = dg.out_degree(i) + starts = [a[0] for a in sorted(starts.items(), key=lambda k: k[1], reverse=True)] + if len(starts) > 1: + o.add_node('starts') + for o in starts: + if o in black_list: + continue + sub(o) + return tree + + def digraph_print(dg, starts=None, sort=False): def digraph_print_sub(node=None, printed=None, level=0): outs = {_: dg.out_degree(_) for _ in dg.successors(node)} From badba967bab70826b00d1782237c0ad2d9a2d502 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 09:06:36 +0300 Subject: [PATCH 36/66] *digraph_tree --- srcxray.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/srcxray.py b/srcxray.py index 49d1b27..86f4423 100755 --- a/srcxray.py +++ b/srcxray.py @@ -419,7 +419,7 @@ def digraph_tree(dg, starts=None): def sub(node): tree.add_node(node) for o in dg.successors(node): - if o in black_list or tree.has_edge(node, o): + if o in black_list or tree.has_node(o): continue tree.add_edge(node,o) sub(o) @@ -430,12 +430,14 @@ def sub(node): for i in [n for (n, d) in dg.in_degree if not d]: starts[i] = dg.out_degree(i) starts = [a[0] for a in sorted(starts.items(), key=lambda k: k[1], reverse=True)] - if len(starts) > 1: - o.add_node('starts') - for o in starts: - if o in black_list: - continue - sub(o) + if len(starts) == 1: + sub(starts[0]) + elif len(starts) > 1: + for o in starts: + if o in black_list: + continue + tree.add_edge('start',o) + sub(o) return tree From 20eee2d1c3377ac5e5a9895079da8035834d2446 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 09:22:36 +0300 Subject: [PATCH 37/66] +default_root --- srcxray.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index 86f4423..022a210 100755 --- a/srcxray.py +++ b/srcxray.py @@ -25,6 +25,7 @@ import glob from pathlib import * +default_root = 'starts' black_list = ('aligned __attribute__ unlikely typeof u32' 'PVOP_CALLEE0 PVOP_VCALLEE0 PVOP_VCALLEE1 if trace_hardirqs_off' 'i NULL likely unlikely true false test_bit NAPI_GRO_CB clear_bit ' @@ -436,7 +437,7 @@ def sub(node): for o in starts: if o in black_list: continue - tree.add_edge('start',o) + tree.add_edge(default_root,o) sub(o) return tree @@ -469,7 +470,7 @@ def digraph_print_sub(node=None, printed=None, level=0): starts[i] = dg.out_degree(i) starts = [a[0] for a in sorted(starts.items(), key=lambda k: k[1], reverse=True)] if len(starts) > 1: - print_limited('starts') + print_limited(default_root) for s in starts: print_limited('\t' + s + ' ->') passed = set() From 33b9bb8e38b01f56980d30e9787b400bc6b72373 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 09:23:00 +0300 Subject: [PATCH 38/66] exit: +digraph_print --- srcxray.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/srcxray.py b/srcxray.py index 022a210..71cf61b 100755 --- a/srcxray.py +++ b/srcxray.py @@ -614,6 +614,8 @@ def main(): else: ret = eval(sys.argv[1] + '(' + ', '.join("'%s'" % (a) for a in sys.argv[2:]) + ')') + if isinstance(ret, nx.DiGraph): + digraph_print(ret) if isinstance(ret, bool) and ret is False: sys.exit(os.EX_CONFIG) if (ret is not None): From d33dcf429413d5efe5bf1209f777f002b8b19902 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 16:01:48 +0300 Subject: [PATCH 39/66] *digraph_print +dst --- srcxray.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/srcxray.py b/srcxray.py index 71cf61b..b204f99 100755 --- a/srcxray.py +++ b/srcxray.py @@ -52,12 +52,13 @@ n = 0 -def print_limited(a): - print(a) +def print_limited(a, out=None): + out = out if out else sys.stdout + out.write(str(a) + '\n') global n n += 1 if n > limit + 1: - print('...') + out.write('...') sys.exit(1) # raise(Exception('Reached limit')) @@ -442,17 +443,21 @@ def sub(node): return tree -def digraph_print(dg, starts=None, sort=False): - def digraph_print_sub(node=None, printed=None, level=0): +def digraph_print(dg, starts=None, dst_fn=None, sort=False): + dst = open(dst_fn, 'w') if dst_fn else None + + def digraph_print_sub(path='', node=None, printed=None, level=0): outs = {_: dg.out_degree(_) for _ in dg.successors(node)} if sort: outs = {a: b for a, b in sorted(outs.items(), key=lambda k: k[1], reverse=True)} if node in printed: - print_limited(level*'\t' + str(node) + ' ^') + print_limited(level*'\t' + str(node) + ' ^', dst) return else: - s = ' ...' if level > level_limit - 2 and outs else '' - print_limited(level*'\t' + str(node) + s) + s = '' + if outs: + s = ' ...' if level > level_limit - 2 else ' @' + path + print_limited(level*'\t' + str(node) + s, dst) printed.add(node) if level > level_limit - 2: return '' @@ -461,7 +466,7 @@ def digraph_print_sub(node=None, printed=None, level=0): if o in passed or o in black_list: continue passed.add(o) - digraph_print_sub(o, printed, level + 1) + digraph_print_sub(path + ' ' + str(node), o, printed, level + 1) printed = set() if not starts: @@ -470,15 +475,19 @@ def digraph_print_sub(node=None, printed=None, level=0): starts[i] = dg.out_degree(i) starts = [a[0] for a in sorted(starts.items(), key=lambda k: k[1], reverse=True)] if len(starts) > 1: - print_limited(default_root) + print_limited(default_root, dst) for s in starts: - print_limited('\t' + s + ' ->') + print_limited('\t' + s + ' ->', dst) passed = set() for o in starts: if o in passed or o in black_list: continue passed.add(o) - digraph_print_sub(o, printed) + if o in dg: + digraph_print_sub('', o, printed) + if dst_fn: + print(dst_fn) + dst.close() def cflow_preprocess(a): From 84a517e09cb95e7ae716c0e5be89e75f816e5ec2 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 16:02:21 +0300 Subject: [PATCH 40/66] space fix --- srcxray.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/srcxray.py b/srcxray.py index b204f99..930be36 100755 --- a/srcxray.py +++ b/srcxray.py @@ -301,9 +301,9 @@ def includes(a): if res and len(res) > 1: r = set() for f in res: - # log('grep " %s \+\(" --include "%s" -r /usr/include/'%(a,f)) + # log('grep " %s \+\(" --include "%s" -r /usr/include/'%(a, f)) # log(os.system( - # 'grep -w "%s" --include "%s" -r /usr/include/'%(a,f))) + # 'grep -w "%s" --include "%s" -r /usr/include/'%(a, f))) if 0 != os.system( 'grep " %s *(" --include "%s" -r /usr/include/ -q' % (a, os.path.basename(f))): @@ -363,7 +363,7 @@ def syscalls(): v.add(syscall) p2 = '/'.join(path[1:]) p2 = m.group(1) - # if log(difflib.get_close_matches(syscall,v) or ''): + # if log(difflib.get_close_matches(syscall, v) or ''): # log(syscall) # log(syscall + ' ' + (includes.get(syscall) or '------')) # man -s 2 timerfd_settime | head -n 20 @@ -382,9 +382,9 @@ def syscalls(): # write_dot to_agraph AGraph # agwrite # srcxray.py 'write_dot(syscalls(), "syscalls.dot")' -# srcxray.py "write_dot(import_cflow(),'a.dot')" +# srcxray.py "write_dot(import_cflow(), 'a.dot')" # write_graphml -# F=sys_mount; srcxray.py "digraph_print(import_cflow(),['$F'])" > $F.tree +# F=sys_mount; srcxray.py "digraph_print(import_cflow(), ['$F'])" > $F.tree # srcxray.py "leaves(read_dot('a.dot'))" # srcxray.py "most_used(read_dot('a.dot'))" # srcxray.py "digraph_print(read_dot('a.dot'))" From f12c8521fa85572b8281a03119b3f225b2823d74 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 16:03:45 +0300 Subject: [PATCH 41/66] fix digraph_tree --- srcxray.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/srcxray.py b/srcxray.py index 930be36..be76ca9 100755 --- a/srcxray.py +++ b/srcxray.py @@ -416,14 +416,16 @@ def starts(dg): # roots return {n: dg.out_degree(n) for (n, d) in dg.in_degree if not d} -def digraph_tree(dg, starts=None): +def digraph_tree(dg, starts=None, black_list=black_list): tree = nx.DiGraph() + def sub(node): tree.add_node(node) for o in dg.successors(node): - if o in black_list or tree.has_node(o): + if o in black_list or tree.has_edge(node, o) or o in starts: + # print(o) continue - tree.add_edge(node,o) + tree.add_edge(node, o) sub(o) printed = set() @@ -438,7 +440,6 @@ def sub(node): for o in starts: if o in black_list: continue - tree.add_edge(default_root,o) sub(o) return tree From a16778e870297aee65d509e999e31d8680f7edde Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 29 Jul 2018 16:04:10 +0300 Subject: [PATCH 42/66] ^cflow_linux --- srcxray.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/srcxray.py b/srcxray.py index be76ca9..4b7c31f 100755 --- a/srcxray.py +++ b/srcxray.py @@ -522,7 +522,7 @@ def cflow_preprocess(a): cflow_param = { "modifier": "__init __inline__ noinline __initdata __randomize_layout __read_mostly asmlinkage " - " __visible __init __leaf__ __ref", + " __visible __init __leaf__ __ref __latent_entropy", "wrapper": "__attribute__ __section__ " "TRACE_EVENT MODULE_AUTHOR MODULE_DESCRIPTION MODULE_LICENSE MODULE_LICENSE MODULE_SOFTDEP " "__acquires __releases __ATTR" @@ -583,21 +583,36 @@ def import_cflow(a=None): def cflow_linux(): - dirs = 'init fs ipc net fs/ext4 net lib block security crypto mm arch/x86/kernel '.split() + dirs = ('init kernel kernel/time ' + 'mm fs fs/ext4 block ' + 'ipc net net/ipv4 ' + 'lib security security/keys crypto ' + 'arch/x86/kernel drivers/char drivers/pci ' + ).split() + # fs/notify/fanotify fs/notify/inotify + all = nx.DiGraph() for a in dirs: + print(a) + dir = nx.DiGraph() for c in glob.glob(os.path.join(a, "*.c")): - dot = str(Path(c).with_suffix(".dot")) - if not os.path.isfile(dot): - g = import_cflow(c) - write_dot(g, dot) - print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) - else: - g = read_dot(dot) - print(dot) - all.add_nodes_from(g.nodes()) - all.add_edges_from(g.edges()) + dot = str(Path(c).with_suffix(".dot")) + if not os.path.isfile(dot): + g = import_cflow(c) + write_dot(g, dot) + print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) + else: + g = read_dot(dot) + # print(dot) + dir.add_nodes_from(g.nodes()) + dir.add_edges_from(g.edges()) + write_dot(dir, os.path.join(a, 'dir.dot')) + digraph_print(digraph_tree(dir), [], os.path.join(a, 'dir.tree')) + all.add_nodes_from(dir.nodes()) + all.add_edges_from(dir.edges()) write_dot(all, 'all.dot') + digraph_print(all, ['x86_64_start_kernel', 'start_kernel', 'main', 'initcall', 'early_param', '__setup'], + 'all.tree') me = os.path.basename(sys.argv[0]) From b2ae6288ef00611a06e0c2775fdca8d30a2cc298 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 31 Jul 2018 05:18:58 +0300 Subject: [PATCH 43/66] +read_dot2, write_dot --- srcxray.py | 100 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 20 deletions(-) diff --git a/srcxray.py b/srcxray.py index 4b7c31f..077a768 100755 --- a/srcxray.py +++ b/srcxray.py @@ -18,12 +18,15 @@ import subprocess import re import networkx as nx -from networkx.drawing.nx_agraph import * +# from networkx.drawing.nx_agraph import read_dot # changes order of successors +from networkx.drawing.nx_pydot import read_dot from networkx.generators.ego import * +from networkx.utils import open_file, make_str from pprint import pprint import difflib import glob from pathlib import * +import pygraphviz default_root = 'starts' black_list = ('aligned __attribute__ unlikely typeof u32' @@ -448,23 +451,24 @@ def digraph_print(dg, starts=None, dst_fn=None, sort=False): dst = open(dst_fn, 'w') if dst_fn else None def digraph_print_sub(path='', node=None, printed=None, level=0): - outs = {_: dg.out_degree(_) for _ in dg.successors(node)} - if sort: - outs = {a: b for a, b in sorted(outs.items(), key=lambda k: k[1], reverse=True)} + if node in black_list: + return if node in printed: print_limited(level*'\t' + str(node) + ' ^', dst) return - else: - s = '' - if outs: - s = ' ...' if level > level_limit - 2 else ' @' + path - print_limited(level*'\t' + str(node) + s, dst) + outs = {_: dg.out_degree(_) for _ in dg.successors(node)} + if sort: + outs = {a: b for a, b in sorted(outs.items(), key=lambda k: k[1], reverse=True)} + s = '' + if outs: + s = ' ...' if level > level_limit - 2 else ' @' + path + print_limited(level*'\t' + str(node) + s, dst) printed.add(node) if level > level_limit - 2: return '' passed = set() for o in outs.keys(): - if o in passed or o in black_list: + if o in passed: continue passed.add(o) digraph_print_sub(path + ' ' + str(node), o, printed, level + 1) @@ -481,11 +485,13 @@ def digraph_print_sub(path='', node=None, printed=None, level=0): print_limited('\t' + s + ' ->', dst) passed = set() for o in starts: - if o in passed or o in black_list: + if o in passed: continue passed.add(o) if o in dg: - digraph_print_sub('', o, printed) + for o in dg.nodes(): + if o not in printed: + digraph_print_sub('', o, printed) if dst_fn: print(dst_fn) dst.close() @@ -507,7 +513,7 @@ def cflow_preprocess(a): s = re.sub(r"SYSCALL_DEFINE[0-9]\((\w*),", r"sys_\1(", s) s = re.sub(r"__setup\(.*,(.*)\)", r"void __setup() {\1();}", s) s = re.sub(r"^(\w*)param\(.*,(.*)\)", r"void \1param() {\2();}", s) - s = re.sub(r"(\w*)initcall\((.*)\)", + s = re.sub(r"^(\w*)initcall\((.*)\)", r"void \1initcall() {\2();}", s) s = re.sub(r"^static ", "", s) # s = re.sub(r"__read_mostly", "", s) @@ -515,6 +521,8 @@ def cflow_preprocess(a): s = re.sub(r"^const ", "", s) s = re.sub(r"^struct (.*) =", r"\1()", s) s = re.sub(r"^struct ", "", s) + s = re.sub(r"\b__initdata\b", "", s) + # s = re.sub(r"__init_or_module", "", s) # __attribute__ # for line in sys.stdin: sys.stdout.write(s) @@ -522,7 +530,7 @@ def cflow_preprocess(a): cflow_param = { "modifier": "__init __inline__ noinline __initdata __randomize_layout __read_mostly asmlinkage " - " __visible __init __leaf__ __ref __latent_entropy", + " __visible __init __leaf__ __ref __latent_entropy __init_or_module ", "wrapper": "__attribute__ __section__ " "TRACE_EVENT MODULE_AUTHOR MODULE_DESCRIPTION MODULE_LICENSE MODULE_LICENSE MODULE_SOFTDEP " "__acquires __releases __ATTR" @@ -555,14 +563,18 @@ def cflow(a): for p in cflow_param.keys()]) + " --include=_sxt --brief --level-indent='0=\t' " + a) + # print(cflow) return popen(cflow) -def import_cflow(a=None): +def import_cflow(a=None, cflow_out=None): cf = my_graph() stack = list() nprev = -1 + cflow_out = open(cflow_out, 'w') if cflow_out else None for line in cflow(a): + if cflow_out: + cflow_out.write(line + '\n') # --print-level m = re.match(r'^([\t]*)([^(^ ^<]+)', str(line)) if m: @@ -582,6 +594,45 @@ def import_cflow(a=None): return cf +def write_dot(g, dot): + dot = str(dot) + dot = open(dot, 'w') + dot.write('strict digraph "None" {\n') + dot.write('rankdir=LR;\n') + dot.write('node [fontname=Ubuntu,shape=none,fontsize=1000];\n') + dot.write('edge [width=1000];\n') + g.remove_nodes_from(black_list) + for n in g.nodes(): + if n == ',': + continue + if not g.out_degree(n): + continue + # dot.write((n if n != 'node' else '"node"') + ';\n') + dot.write((n if n != 'node' else '"node"') + ' -> { ') + dot.write(' '.join([str(a) if a != 'node' else '"node"' for a in g.successors(n) if str(a) != ','])) + dot.write(' };\n') + dot.write('}\n') + dot.close() + print(dot.name) + + +@open_file(0, mode='r') +def read_dot2(dot): + # read_dot pydot.graph_from_dot_data parse_dot_data from_pydot + dg = nx.DiGraph() + for a in dot: + if '->' in a: + m = re.match('^(.+) -> {(.+)}', a) + if m: + dg.add_edges_from([(m.group(1), b) for b in m.group(2).split()]) + else: + m = re.match('(.+) -> (.*);', a) + if m: + dg.add_edge(m.group(1), m.group(2)) + print(m.group(1), m.group(2)) + return dg + + def cflow_linux(): dirs = ('init kernel kernel/time ' 'mm fs fs/ext4 block ' @@ -596,19 +647,28 @@ def cflow_linux(): print(a) dir = nx.DiGraph() for c in glob.glob(os.path.join(a, "*.c")): + g = None dot = str(Path(c).with_suffix(".dot")) if not os.path.isfile(dot): - g = import_cflow(c) + g = import_cflow(c, Path(c).with_suffix(".cflow")) + print(g['start_kernel']) write_dot(g, dot) print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) else: - g = read_dot(dot) + try: + g = read_dot(dot) + except TypeError: + g = nx.drawing.nx_agraph.read_dot(dot) + g2 = nx.DiGraph() + g2.add_edges_from(g.edges()) + write_dot(g, dot + '2') # print(dot) - dir.add_nodes_from(g.nodes()) + digraph_print(g, [], Path(c).with_suffix(".tree")) + # dir.add_nodes_from(g.nodes()) dir.add_edges_from(g.edges()) - write_dot(dir, os.path.join(a, 'dir.dot')) + write_dot(dir, str(os.path.join(a, 'dir.dot'))) digraph_print(digraph_tree(dir), [], os.path.join(a, 'dir.tree')) - all.add_nodes_from(dir.nodes()) + # all.add_nodes_from(dir.nodes()) all.add_edges_from(dir.edges()) write_dot(all, 'all.dot') digraph_print(all, ['x86_64_start_kernel', 'start_kernel', 'main', 'initcall', 'early_param', '__setup'], From d2eb106b4340e78223f142f72f606a639ad10310 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 31 Jul 2018 05:21:50 +0300 Subject: [PATCH 44/66] ^cflow_linux --- srcxray.py | 79 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/srcxray.py b/srcxray.py index 077a768..76cbda6 100755 --- a/srcxray.py +++ b/srcxray.py @@ -635,44 +635,57 @@ def read_dot2(dot): def cflow_linux(): dirs = ('init kernel kernel/time ' - 'mm fs fs/ext4 block ' - 'ipc net net/ipv4 ' - 'lib security security/keys crypto ' + 'fs fs/ext4 block ' + 'ipc net ' + 'lib security security/keys ' 'arch/x86/kernel drivers/char drivers/pci ' ).split() + + # dirs += ('mm net/ipv4 crypto').split() + dirs = ('init kernel arch/x86/kernel fs ').split() + # fs/notify/fanotify fs/notify/inotify - all = nx.DiGraph() - for a in dirs: - print(a) - dir = nx.DiGraph() - for c in glob.glob(os.path.join(a, "*.c")): - g = None - dot = str(Path(c).with_suffix(".dot")) - if not os.path.isfile(dot): - g = import_cflow(c, Path(c).with_suffix(".cflow")) - print(g['start_kernel']) - write_dot(g, dot) - print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) - else: - try: - g = read_dot(dot) - except TypeError: - g = nx.drawing.nx_agraph.read_dot(dot) - g2 = nx.DiGraph() - g2.add_edges_from(g.edges()) - write_dot(g, dot + '2') - # print(dot) - digraph_print(g, [], Path(c).with_suffix(".tree")) - # dir.add_nodes_from(g.nodes()) - dir.add_edges_from(g.edges()) - write_dot(dir, str(os.path.join(a, 'dir.dot'))) - digraph_print(digraph_tree(dir), [], os.path.join(a, 'dir.tree')) - # all.add_nodes_from(dir.nodes()) - all.add_edges_from(dir.edges()) - write_dot(all, 'all.dot') - digraph_print(all, ['x86_64_start_kernel', 'start_kernel', 'main', 'initcall', 'early_param', '__setup'], + try: + print('loading all.dot') + all = read_dot2('all.dot') + # all = nx.DiGraph(read_dot('all.dot')) + except FileNotFoundError: + all = nx.DiGraph() + for a in dirs: + print(a) + index = nx.DiGraph() + for c in glob.glob(os.path.join(a, "*.c")): + g = None + dot = str(Path(c).with_suffix(".dot")) + if not os.path.isfile(dot): + g = import_cflow(c, Path(c).with_suffix(".cflow")) + write_dot(g, dot) + print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) + else: + print(dot) + try: + # g = nx.drawing.nx_agraph.read_dot(dot) + g = read_dot(dot) + except (TypeError, pygraphviz.agraph.DotError): + print('nx_pydot <- nx_agraph') + g = nx.drawing.nx_pydot.read_dot(dot) + # digraph_print(g, [], Path(c).with_suffix(".tree")) + # index.add_nodes_from(g.nodes()) + index.add_edges_from(g.edges()) + write_dot(index, str(os.path.join(a, 'index.dot'))) + digraph_print(digraph_tree(index), [], os.path.join(a, 'index.tree')) + # all.add_nodes_from(index.nodes()) + all.add_edges_from(index.edges()) + write_dot(all, 'all.dot') + # print('loops: ' + str( all.nodes_with_selfloops())) + print('trees:') + digraph_print(all, ['x86_64_start_kernel', 'start_kernel', 'main', 'initcall', 'early_param', '__setup', 'sys_write', 'write'], 'all.tree') + start_kernel = digraph_tree(all, ['start_kernel']) + write_dot(start_kernel, 'start_kernel.dot') + write_dot(reduce_graph(start_kernel), 'start_kernel-reduced.dot') + write_dot(reduce_graph(reduce_graph(start_kernel)), 'start_kernel-reduced2.dot') me = os.path.basename(sys.argv[0]) From f3aa4cace0111c22e0a01a9b47688d84eac0eefe Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 05:50:37 +0300 Subject: [PATCH 45/66] fix spaces --- srcxray.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/srcxray.py b/srcxray.py index 76cbda6..57f6db6 100755 --- a/srcxray.py +++ b/srcxray.py @@ -29,20 +29,20 @@ import pygraphviz default_root = 'starts' -black_list = ('aligned __attribute__ unlikely typeof u32' - 'PVOP_CALLEE0 PVOP_VCALLEE0 PVOP_VCALLEE1 if trace_hardirqs_off' +black_list = ('aligned __attribute__ unlikely typeof u32 ' + 'PVOP_CALLEE0 PVOP_VCALLEE0 PVOP_VCALLEE1 if trace_hardirqs_off ' 'i NULL likely unlikely true false test_bit NAPI_GRO_CB clear_bit ' - 'atomic_read preempt_disable preempt_enable container_of ENOSYS ' + 'atomic_read preempt_disable preempt_enable container_of ENOSYS ' 'READ_ONCE u64 u8 _RET_IP_ ret current ' 'AT_FDCWD fdput EBADF file_inode ' 'ssize_t path_put __user ' - 'list_empty memcpy size_t loff_t pos d_inode dput copy_to_user EIO bool out IS_ERR ' + 'list_empty memcpy size_t loff_t pos d_inode dput copy_to_user EIO bool out IS_ERR ' 'EPERM rcu_read_lock rcu_read_unlock spin_lock spin_unlock list_for_each_entry kfree ' 'GFP_KERNEL ENOMEM EFAULT ENOENT EAGAIN PTR_ERR PAGE_SHIFT PAGE_SIZE ' 'pgoff_t pte_t pmd_t HPAGE_PMD_NR PageLocked entry swp_entry_t next unlock_page spinlock_t end start ' ' VM_BUG_ON VM_BUG_ON_PAGE BDI_SHOW max ' 'ssize_t path_put __user ' - 'list_del compound_head list_add cond_resched put_page nr_pages min spin_lock_irqsave IS_ENABLED ' + 'list_del compound_head list_add cond_resched put_page nr_pages min spin_lock_irqsave IS_ENABLED ' 'EBUSY UL NODE_DATA pr_err memset list size ptl PAGE_MASK pr_info offset addr get_page sprintf ' 'INIT_LIST_HEAD NUMA_NO_NODE spin_unlock_irqrestore mutex_unlock mutex_lock ' 'page_to_nid page_to_pfn pfn page_zone pfn_to_page' @@ -335,7 +335,7 @@ def syscalls(): if not os.path.isfile(scd): os.system("grep SYSCALL_DEFINE -r --include='*.c' > " + scd) with open(scd, 'r') as f: - v = set('sigsuspend', 'llseek', 'sysfs', 'sync_file_range2', 'ustat', 'bdflush') + v = set(['sigsuspend', 'llseek', 'sysfs', 'sync_file_range2', 'ustat', 'bdflush']) for s in f: if any(x in s.lower() for x in ['compat', 'stub']): continue @@ -529,8 +529,8 @@ def cflow_preprocess(a): cflow_param = { - "modifier": "__init __inline__ noinline __initdata __randomize_layout __read_mostly asmlinkage " - " __visible __init __leaf__ __ref __latent_entropy __init_or_module ", + "modifier": "__init __inline__ noinline __initdata __randomize_layout asmlinkage " + " __visible __init __leaf__ __ref __latent_entropy __init_or_module ", "wrapper": "__attribute__ __section__ " "TRACE_EVENT MODULE_AUTHOR MODULE_DESCRIPTION MODULE_LICENSE MODULE_LICENSE MODULE_SOFTDEP " "__acquires __releases __ATTR" From faea156d8fc2a720d4eb28b424cab081245c24a4 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 05:53:42 +0300 Subject: [PATCH 46/66] updates --- srcxray.py | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/srcxray.py b/srcxray.py index 57f6db6..9184066 100755 --- a/srcxray.py +++ b/srcxray.py @@ -44,10 +44,19 @@ 'ssize_t path_put __user ' 'list_del compound_head list_add cond_resched put_page nr_pages min spin_lock_irqsave IS_ENABLED ' 'EBUSY UL NODE_DATA pr_err memset list size ptl PAGE_MASK pr_info offset addr get_page sprintf ' - 'INIT_LIST_HEAD NUMA_NO_NODE spin_unlock_irqrestore mutex_unlock mutex_lock ' - 'page_to_nid page_to_pfn pfn page_zone pfn_to_page' - 'BUG BUG_ON flags WARN_ON_ONCE ENODEV cpu_to_le16 cpumask_bits ' - 'ERR_PTR ENOTSUPP EOPNOTSUPP EOPNOTSUPP WARN_ON EINVAL ').split() + 'INIT_LIST_HEAD NUMA_NO_NODE spin_unlock_irqrestore mutex_unlock mutex_lock ' + 'page_to_nid page_to_pfn pfn page_zone pfn_to_page ' + 'BUG BUG_ON flags WARN_ON_ONCE ENODEV cpu_to_le16 cpumask_bits ' + 'ERR_PTR ENOTSUPP EOPNOTSUPP EOPNOTSUPP WARN_ON EINVAL i name ' + 'sigset_t fdget put_user get_user copy_from_user LOOKUP_FOLLOW LOOKUP_EMPTY EINTR ' + 'O_CLOEXEC err getname access_ok task_pid_vnr cred ' + 'percpu_ref_put get_timespec64 sigdelsetmask ns_capable kzalloc capable f_mode O_LARGEFILE pos_from_hilo ' + 'pr_debug error current_cred ESRCH f_path find_task_by_vpid ' + 'retry LOOKUP_REVAL retry_estale user_path_at lookup_flags old ' + 'current_user_ns spin_lock_irq spin_unlock_irq prepare_creds ' + 'tasklist_lock commit_creds read_lock read_unlock SIGKILL SIGSTOP abort_creds fd_install ' + 'real_mount FMODE_WRITE tv_nsec putname ' + ).split() level_limit = 8 @@ -107,7 +116,7 @@ def func_referers_git_grep(name): r = None for line in popen(r'git grep --no-index --word-regexp --show-function ' r'"^\s.*\b%s" ' - r'**.\[hc\] **.cpp **.cc **.hh' % (name)): + r'**.\[hc\] **.cpp **.cc **.hh || true' % (name)): # Filter out names in comment afer function, # when comment start from ' *' # To see the problem try "git grep -p and" @@ -285,8 +294,7 @@ def includes(a): res = [] # log(a) for a in popen('man -s 2 %s 2> /dev/null |' - ' head -n 20 | grep include || true' % (a), - shell=True): + ' head -n 20 | grep include || true' % (a)): m = re.match('.*<(.*)>', a) if m: res.append(m.group(1)) @@ -370,14 +378,14 @@ def syscalls(): # log(syscall) # log(syscall + ' ' + (includes.get(syscall) or '------')) # man -s 2 timerfd_settime | head -n 20 - # sc.add_edge('syscalls', path[0] + '/') - # sc.add_edge(path[0] + '/', p2) - # sc.add_edge(p2, syscall) - i = includes(syscall) - log(p2 + ' ' + str(i) + ' ' + syscall) - sc.add_edge(i, i+' - '+p2) - sc.add_edge(i+' - '+p2, syscall) - # sc.add_edge(includes(syscall), syscall) + if False: + i = includes(syscall) + log(p2 + ' ' + str(i) + ' ' + syscall) + sc.add_edge(i, i+' - '+p2) + sc.add_edge(i+' - '+p2, 'sys_' + syscall) + else: + sc.add_edge(path[0] + '/', p2) + sc.add_edge(p2, 'sys_' + syscall) return sc @@ -391,11 +399,14 @@ def syscalls(): # srcxray.py "leaves(read_dot('a.dot'))" # srcxray.py "most_used(read_dot('a.dot'))" # srcxray.py "digraph_print(read_dot('a.dot'))" -# srcxray.py "digraph_print(reduce_graph(reduce_graph(read_dot('a.dot'))))" +# srcxray.py "write_dot(reduce_graph(read_dot('no-loops.dot')),'reduced.dot')" # srcxray.py "pprint(most_used(read_dot('a.dot')))" +# srcxray.py "write_dot(digraph_tree(read_dot2('all.dot'), ['sys_clone']), 'sys_clone.dot')" +# srcxray.py "write_dot(add_rank('reduced.dot'), 'ranked.dot')" +# srcxray.py "write_dot(remove_loops(read_dot2('reduced.dot')), 'no-loops.dot')" def cleanup(a): - dg = read_dot(a) + dg = read_dot2(a) print(dg.number_of_edges()) dg.remove_nodes_from(black_list) print(dg.number_of_edges()) From 26695d4150dbd02eb7bfc29b5e7540aa8ce53941 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 08:04:46 +0300 Subject: [PATCH 47/66] *reduce_graph --- srcxray.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/srcxray.py b/srcxray.py index 9184066..3cbde7e 100755 --- a/srcxray.py +++ b/srcxray.py @@ -281,12 +281,13 @@ def my_graph(name=None): return g -def reduce_graph(g): +def reduce_graph(g, m=None): rm = set() - for e in g: - if not g.out_degree(e): - rm.add(e) + m = g.number_of_nodes() if not m else m + print(g.number_of_edges()) + rm = [n for (n, d) in g.out_degree if not d and g.in_degree(n) <= m] g.remove_nodes_from(rm) + print(g.number_of_edges()) return g From 1163ea990ca2a02411fac406f7ed95f3e888d3e6 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 08:05:52 +0300 Subject: [PATCH 48/66] +sort_dict --- srcxray.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/srcxray.py b/srcxray.py index 3cbde7e..f881bd4 100755 --- a/srcxray.py +++ b/srcxray.py @@ -419,6 +419,10 @@ def leaves(dg): return {n: dg.in_degree(n) for (n, d) in dg.out_degree if not d} +def sort_dict(d): + return [a for a, b in sorted(d.items(), key=lambda k: k[1], reverse=True)] + + def most_used(dg, ins=10, outs=10): # return {a: b for a, b in sorted(dg.in_degree, key=lambda k: k[1]) if b > 1 and} # return [(x, dg.in_degree(x), dg.out_degree(x)) From 1dd2f855e5e64106b1e15a0d123138b236355920 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 08:55:42 +0300 Subject: [PATCH 49/66] +to_dg --- srcxray.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index f881bd4..f37edde 100755 --- a/srcxray.py +++ b/srcxray.py @@ -407,7 +407,7 @@ def syscalls(): # srcxray.py "write_dot(remove_loops(read_dot2('reduced.dot')), 'no-loops.dot')" def cleanup(a): - dg = read_dot2(a) + g = to_dg(a) print(dg.number_of_edges()) dg.remove_nodes_from(black_list) print(dg.number_of_edges()) @@ -649,6 +649,13 @@ def read_dot2(dot): return dg +def to_dg(a): + if isinstance(a, nx.DiGraph): + return a + if os.path.isfile(a): + return read_dot2(a) + + def cflow_linux(): dirs = ('init kernel kernel/time ' 'fs fs/ext4 block ' From d747641520cd71a2722b3286a456f4b5cf6690eb Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 08:59:50 +0300 Subject: [PATCH 50/66] +stats --- srcxray.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/srcxray.py b/srcxray.py index f37edde..5cc5ec3 100755 --- a/srcxray.py +++ b/srcxray.py @@ -15,12 +15,14 @@ import os import sys import collections +from munch import * import subprocess import re import networkx as nx # from networkx.drawing.nx_agraph import read_dot # changes order of successors from networkx.drawing.nx_pydot import read_dot from networkx.generators.ego import * +from networkx.algorithms.dag import * from networkx.utils import open_file, make_str from pprint import pprint import difflib @@ -711,6 +713,44 @@ def cflow_linux(): write_dot(reduce_graph(reduce_graph(start_kernel)), 'start_kernel-reduced2.dot') +def stats(a): + dg = to_dg(a) + stat = Munch() + im = dict() + om = dict() + leaves = set() + roots = set() + stat.edge_nodes = 0 + for n in dg: + id = dg.in_degree(n) + od = dg.out_degree(n) + if id == 1 and od == 1: + stat.edge_nodes += 1 + if id: + im[n] = id + else: + roots.add(n) + if od: + om[n] = od + else: + leaves.add(n) + stat.max_in_degree = max(dict(dg.in_degree).values()) + stat.max_out_degree = max(dict(dg.out_degree).values()) + stat.leaves = len(leaves) + stat.roots = len(roots) + # pprint(im) + # pprint(om) + stat._popular = ' '.join(sort_dict(im)[:10]) + stat._biggest = ' '.join(sort_dict(om)[:10]) + gd = remove_loops(dg) + stat.dag_longest_path_len = len(dag_longest_path(dg)) + print(' '.join(dag_longest_path(dg))) + for a in [nx.DiGraph.number_of_nodes, nx.DiGraph.number_of_edges, nx.DiGraph.number_of_selfloops, + nx.DiGraph.order]: + stat[a.__name__] = a(dg) + pprint(dict(stat)) + + me = os.path.basename(sys.argv[0]) From 95881dcf91badeab490fc45f6af4c0f13534a61a Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:03:32 +0300 Subject: [PATCH 51/66] *cflow --- srcxray.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/srcxray.py b/srcxray.py index 5cc5ec3..d7e7702 100755 --- a/srcxray.py +++ b/srcxray.py @@ -560,11 +560,15 @@ def cflow_preprocess(a): def cflow(a): - arg = a + if os.path.isfile('include/linux/cache.h'): + for m in popen("ctags -x --c-kinds=d include/linux/cache.h | cut -d' ' -f 1 | sort -u"): + if m in cflow_param['modifier']: + print(m) + else: + cflow_param['modifier'] += ' ' + a if not a: # arg = "$(find -name '*.[ch]' -o -name '*.cpp' -o -name '*.hh')" - arg = "*.c *.h *.cpp *.hh " - arg = " $(cat cscope.files)" + a = "$(cat cscope.files)" if os.path.isfile('cscope.files') else "*.c *.h *.cpp *.hh " elif isinstance(a, list): pass elif os.path.isdir(a): From ed719c3353a260bf2eb117353bcd03ce47516cff Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:04:10 +0300 Subject: [PATCH 52/66] read_dot2: +{} --- srcxray.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/srcxray.py b/srcxray.py index d7e7702..d41e685 100755 --- a/srcxray.py +++ b/srcxray.py @@ -643,15 +643,18 @@ def read_dot2(dot): # read_dot pydot.graph_from_dot_data parse_dot_data from_pydot dg = nx.DiGraph() for a in dot: + a = a.strip() if '->' in a: - m = re.match('^(.+) -> {(.+)}', a) + m = re.match('"?([^"]+)"? -> {(.+)}', a) if m: - dg.add_edges_from([(m.group(1), b) for b in m.group(2).split()]) + dg.add_edges_from([(m.group(1), b.strip('"')) for b in m.group(2).split() if b != m.group(1)]) else: - m = re.match('(.+) -> (.*);', a) + m = re.match('"?([^"]+)"? -> "?([^"]*)"?;', a) if m: - dg.add_edge(m.group(1), m.group(2)) - print(m.group(1), m.group(2)) + if m.group(1) != m.group(2): + dg.add_edge(m.group(1), m.group(2)) + else: + log(a) return dg From d3cb46177f907457491e23c1958cdb6b9b6cd1d3 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:06:07 +0300 Subject: [PATCH 53/66] write_dot: +rank --- srcxray.py | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/srcxray.py b/srcxray.py index d41e685..39784db 100755 --- a/srcxray.py +++ b/srcxray.py @@ -616,23 +616,55 @@ def import_cflow(a=None, cflow_out=None): return cf +def rank(g, n): + try: + if g.nodes[n]['rank1'] < abs(g.nodes[n]['rank2']): + return g.nodes[n]['rank1'] + else: + return g.__dict__['max_rank'] + 1 + g.nodes[n]['rank2'] + except KeyError: + return None + + def write_dot(g, dot): dot = str(dot) dot = open(dot, 'w') dot.write('strict digraph "None" {\n') - dot.write('rankdir=LR;\n') - dot.write('node [fontname=Ubuntu,shape=none,fontsize=1000];\n') - dot.write('edge [width=1000];\n') + dot.write('rankdir=LR\nnodesep=0\n') + # dot.write('ranksep=50\n') + dot.write('node [fontname=Ubuntu,shape=none];\n') + # dot.write('edge [width=10000];\n') + dot.write('edge [width=1];\n') g.remove_nodes_from(black_list) + ranks = collections.defaultdict(list) for n in g.nodes(): if n == ',': continue + r = rank(g, n) + if r: + ranks[r].append(n) if not g.out_degree(n): continue # dot.write((n if n != 'node' else '"node"') + ';\n') - dot.write((n if n != 'node' else '"node"') + ' -> { ') - dot.write(' '.join([str(a) if a != 'node' else '"node"' for a in g.successors(n) if str(a) != ','])) - dot.write(' };\n') + # dot.write((n if n != 'node' else '"node"') + ' -> { ') + dot.write('"%s" -> { ' % (n)) + # dot.write(' '.join([str(a) if a != 'node' else '"node"' for a in g.successors(n) if str(a) != ','])) + dot.write(' '.join(['"%s"' % (str(a)) for a in g.successors(n) if str(a) != ','])) + if r and scaled: + dot.write(' } [penwidth=%d label=%d];\n' % (100/r, r)) + else: + dot.write(' } ;\n') + print(ranks.keys()) + for r in ranks.keys(): + dot.write("{ rank=same %s }\n" % (' '.join(['"%s"' % (str(a)) for a in ranks[r]]))) + for n in g.nodes(): + prop = Munch() + if scaled and len(ranks): + prop.fontsize = 500 + 10000 / (len(ranks[rank(g, n)]) + 1) + # prop.label = n + ' ' + str(rank(g,n)) + dot.write('"%s" [%s]\n' % (n, ','.join(['%s="%s"' % (a, str(prop[a])) for a in prop]))) + # else: + # dot.write('"%s"\n'%(n)) dot.write('}\n') dot.close() print(dot.name) From d62708ae1052d5b03d2292d03ab210d3e3499f9d Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:07:08 +0300 Subject: [PATCH 54/66] +remove_loops --- srcxray.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/srcxray.py b/srcxray.py index 39784db..8675259 100755 --- a/srcxray.py +++ b/srcxray.py @@ -697,6 +697,30 @@ def to_dg(a): return read_dot2(a) +def remove_loops(dg): + rm = [] + visited = set() + path = [object()] + path_set = set(path) + stack = [iter(dg)] + while stack: + for v in stack[-1]: + if v in path_set: + rm.append((path[-1], v)) + elif v not in visited: + visited.add(v) + path.append(v) + path_set.add(v) + stack.append(iter(dg[v])) + break + else: + path_set.remove(path.pop()) + stack.pop() + # print(rm) + dg.remove_edges_from(rm) + return dg + + def cflow_linux(): dirs = ('init kernel kernel/time ' 'fs fs/ext4 block ' From ae13533730e003c3310f3695a48a655c0f6cef60 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:11:55 +0300 Subject: [PATCH 55/66] +cflow_dir --- srcxray.py | 53 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/srcxray.py b/srcxray.py index 8675259..8ba64d0 100755 --- a/srcxray.py +++ b/srcxray.py @@ -721,6 +721,31 @@ def remove_loops(dg): return dg +def cflow_dir(a): + index = nx.DiGraph() + for c in glob.glob(os.path.join(a, "*.c")): + g = None + dot = str(Path(c).with_suffix(".dot")) + if not os.path.isfile(dot): + g = import_cflow(c, Path(c).with_suffix(".cflow")) + write_dot(g, dot) + print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) + else: + print(dot) + try: + # g = nx.drawing.nx_agraph.read_dot(dot) + g = read_dot(dot) + except (TypeError, pygraphviz.agraph.DotError): + print('nx_pydot <- nx_agraph') + g = nx.drawing.nx_pydot.read_dot(dot) + # digraph_print(g, [], Path(c).with_suffix(".tree")) + # index.add_nodes_from(g.nodes()) + index.add_edges_from(g.edges()) + write_dot(index, str(os.path.join(a, 'index.dot'))) + digraph_print(digraph_tree(index), [], os.path.join(a, 'index.tree')) + return index + + def cflow_linux(): dirs = ('init kernel kernel/time ' 'fs fs/ext4 block ' @@ -742,33 +767,15 @@ def cflow_linux(): all = nx.DiGraph() for a in dirs: print(a) - index = nx.DiGraph() - for c in glob.glob(os.path.join(a, "*.c")): - g = None - dot = str(Path(c).with_suffix(".dot")) - if not os.path.isfile(dot): - g = import_cflow(c, Path(c).with_suffix(".cflow")) - write_dot(g, dot) - print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) - else: - print(dot) - try: - # g = nx.drawing.nx_agraph.read_dot(dot) - g = read_dot(dot) - except (TypeError, pygraphviz.agraph.DotError): - print('nx_pydot <- nx_agraph') - g = nx.drawing.nx_pydot.read_dot(dot) - # digraph_print(g, [], Path(c).with_suffix(".tree")) - # index.add_nodes_from(g.nodes()) - index.add_edges_from(g.edges()) - write_dot(index, str(os.path.join(a, 'index.dot'))) - digraph_print(digraph_tree(index), [], os.path.join(a, 'index.tree')) + index = cflow_dir(a) # all.add_nodes_from(index.nodes()) all.add_edges_from(index.edges()) write_dot(all, 'all.dot') - # print('loops: ' + str( all.nodes_with_selfloops())) + remove_loops(all) + print('loops: ' + str(list(all.nodes_with_selfloops()))) print('trees:') - digraph_print(all, ['x86_64_start_kernel', 'start_kernel', 'main', 'initcall', 'early_param', '__setup', 'sys_write', 'write'], + digraph_print(all, ['x86_64_start_kernel', 'start_kernel', 'main', 'initcall', 'early_param', + '__setup', 'sys_write', 'write'], 'all.tree') start_kernel = digraph_tree(all, ['start_kernel']) write_dot(start_kernel, 'start_kernel.dot') From 2b4a2dc992f6b71d8eb4d05ff607d6e25f63f98c Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:12:22 +0300 Subject: [PATCH 56/66] +dot_expand --- srcxray.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/srcxray.py b/srcxray.py index 8ba64d0..44c4703 100755 --- a/srcxray.py +++ b/srcxray.py @@ -821,6 +821,13 @@ def stats(a): pprint(dict(stat)) +def dot_expand(a, b): + a = to_dg(a) + b = to_dg(b) + a.add_edges_from(list(b.out_edges(b.nbunch_iter(a.nodes())))) + return a + + me = os.path.basename(sys.argv[0]) From d17f89a41937f4e4d35c70efff5fc6b85d0d1ab0 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:13:26 +0300 Subject: [PATCH 57/66] +add_rank --- srcxray.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index 44c4703..13521be 100755 --- a/srcxray.py +++ b/srcxray.py @@ -65,6 +65,8 @@ limit = 100000 n = 0 +scaled = False + def print_limited(a, out=None): out = out if out else sys.stdout @@ -478,8 +480,11 @@ def digraph_print_sub(path='', node=None, printed=None, level=0): if sort: outs = {a: b for a, b in sorted(outs.items(), key=lambda k: k[1], reverse=True)} s = '' + if 'rank' in dg.nodes[node]: + s = str(dg.nodes[node]['rank']) + ranks[dg.nodes[node]['rank']].append(node) if outs: - s = ' ...' if level > level_limit - 2 else ' @' + path + s += ' ...' if level > level_limit - 2 else ' @' + path print_limited(level*'\t' + str(node) + s, dst) printed.add(node) if level > level_limit - 2: @@ -828,6 +833,37 @@ def dot_expand(a, b): return a +def add_rank(g): + g= to_dg(g) + passed1 = set() + passed2 = set() + rn1 = 1 + rn2 = -1 + r1 = [n for (n, d) in g.in_degree if not d] + r2 = [n for (n, d) in g.out_degree if not d] + while r1 or r2: + if r1: + nxt = set() + for n in r1: + g.nodes[n]['rank1'] = max(rn1, g.nodes[n].get('rank1', rn1)) + for i in [_ for _ in g.successors(n)]: + nxt.add(i) + passed1.add(i) + rn1 += 1 + r1 = nxt + if r2: + nxt = set() + for n in r2: + g.nodes[n]['rank2'] = min(rn2, g.nodes[n].get('rank2', rn2)) + for i in [_ for _ in g.predecessors(n)]: + nxt.add(i) + passed2.add(i) + rn2 -= 1 + r2 = nxt + g.__dict__['max_rank'] = rn1 + return g + + me = os.path.basename(sys.argv[0]) From a041e78173137f78edd36f13f832fd4ee9914cdc Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Tue, 7 Aug 2018 09:14:44 +0300 Subject: [PATCH 58/66] +#exec --- srcxray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/srcxray.py b/srcxray.py index 13521be..a73e662 100755 --- a/srcxray.py +++ b/srcxray.py @@ -885,6 +885,7 @@ def main(): else: if '(' in sys.argv[1]: ret = eval(sys.argv[1]) + # ret = exec(sys.argv[1]) else: ret = eval(sys.argv[1] + '(' + ', '.join("'%s'" % (a) for a in sys.argv[2:]) + ')') From 986a091dec6c6f8157bf935d473b782a4f24bd74 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Wed, 8 Aug 2018 09:03:47 +0300 Subject: [PATCH 59/66] set -> uniq idom set is unstable --- srcxray.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/srcxray.py b/srcxray.py index a73e662..bbab051 100755 --- a/srcxray.py +++ b/srcxray.py @@ -116,9 +116,9 @@ def extract_referer_test(): def func_referers_git_grep(name): - res = set() + res = list() r = None - for line in popen(r'git grep --no-index --word-regexp --show-function ' + for line in popen(r'git grep --threads 1 --no-index --word-regexp --show-function ' r'"^\s.*\b%s" ' r'**.\[hc\] **.cpp **.cc **.hh || true' % (name)): # Filter out names in comment afer function, @@ -133,7 +133,7 @@ def func_referers_git_grep(name): r = None break if r and r != name and r not in black_list: - res.add(r) + res.append(r) r = None r = extract_referer(line) return res @@ -149,15 +149,15 @@ def func_referers_cscope(name): print("Recommended: cscope -bkR", file=sys.stderr) cscope_warned = True return [] - res = set([l.split()[1] for l in popen(r'cscope -d -L3 "%s"' % - (name)) if l not in black_list]) + res = list(dict.fromkeys([l.split()[1] for l in popen(r'cscope -d -L3 "%s"' % + (name)) if l not in black_list])) if not res: res = func_referers_git_grep(name) return res def func_referers_all(name): - return set(func_referers_git_grep(name) + func_referers_cscope(name)) + return list(dict.fromkeys(func_referers_git_grep(name) + func_referers_cscope(name))) def referers_tree(name, referer=None, printed=None, level=0): @@ -204,13 +204,10 @@ def referers_dep(name, referer=None, printed=None, level=0): return if level > level_limit - 2: return '' - referers = set(referer(name)) + referers = referer(name) if referers: printed.add(name) - print(name, end=': ') - for a in referers: - print(a, end=' ') - print() + print("%s:" % (name), ' '.join(referers)) for a in referers: referers_dep(a, referer, printed, level + 1) else: @@ -256,20 +253,17 @@ def call_dep(node, printed=None, level=0): printed = set() if node in printed: return - calls = set() + calls = list() for a in [line.split()[1] for line in popen('cscope -d -L2 "%s"' % (node))]: if a in black_list: continue - calls.add(a) + calls.append(a) if calls: if level < level_limit - 1: printed.add(node) - print(node, end=': ') - for a in calls: - print(a, end=' ') - print() - for a in calls: + print("%s:" % (node), ' '.join(list(dict.fromkeys(calls)))) + for a in list(dict.fromkeys(calls)): call_dep(a, printed, level + 1) else: pass From 92d8b73bb3492e6aa4f801dd0dac8dfef0451875 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Wed, 8 Aug 2018 09:08:05 +0300 Subject: [PATCH 60/66] unittest --- srcxray.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/srcxray.py b/srcxray.py index bbab051..5eb1a02 100755 --- a/srcxray.py +++ b/srcxray.py @@ -29,6 +29,8 @@ import glob from pathlib import * import pygraphviz +import unittest +import types default_root = 'starts' black_list = ('aligned __attribute__ unlikely typeof u32 ' @@ -866,29 +868,49 @@ def usage(): print(me, c, "") print("Try this:") print("cd linux/init") + print(me, "unittest") print(me, "referers_tree nfs_root_data") print(me, "call_tree start_kernel") print(me, "Emergency termination: ^Z, kill %1") +class _unittest_autotest(unittest.TestCase): + def test_1(self): + self.assertTrue(os.path.isdir('include/linux/')) + os.chdir('init') + self.assertEqual('\t\t\t\t\tprepare_namespace ^', popen('srcxray.py referers_tree nfs_root_data')[-1]) + self.assertEqual('initrd_load: prepare_namespace', popen('srcxray.py referers_dep nfs_root_data')[-1]) + self.assertEqual('calibrate_delay_converge: __delay', popen('srcxray.py call_dep start_kernel')[-2]) + self.assertEqual('\t\tcpu_startup_entry', popen('srcxray.py call_tree start_kernel')[-1]) + os.chdir('..') + self.assertTrue(syscalls().number_of_edges() > 400) + # digraph_print: + self.assertEqual("\thandle_initrd ^", popen("srcxray.py import_cflow init/do_mounts_initrd.c")[-1]) + self.assertEqual("\t\t4", popen('srcxray.py "nx.DiGraph([{1,2},{2,3},{2,4}])"')[-1]) + + def main(): try: ret = False if len(sys.argv) == 1: print('Run', me, 'usage') else: - if '(' in sys.argv[1]: - ret = eval(sys.argv[1]) + a1 = sys.argv[1] + sys.argv = sys.argv[1:] + if isinstance(eval(a1), types.ModuleType): + ret = eval(a1+".main()") + elif '(' in a1: + ret = eval(a1) # ret = exec(sys.argv[1]) else: - ret = eval(sys.argv[1] + '(' + ', '.join("'%s'" % (a) - for a in sys.argv[2:]) + ')') + ret = eval(a1 + '(' + ', '.join("'%s'" % (a1) + for a1 in sys.argv[1:]) + ')') if isinstance(ret, nx.DiGraph): digraph_print(ret) if isinstance(ret, bool) and ret is False: sys.exit(os.EX_CONFIG) - if (ret is not None): - print(ret) + # if (ret is not None): + # print(ret) except KeyboardInterrupt: log("\nInterrupted") From 4b61215d1263e646d78fb08aa98a2ae973977977 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Wed, 8 Aug 2018 09:08:20 +0300 Subject: [PATCH 61/66] *usage --- srcxray.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index 5eb1a02..4c2fe71 100755 --- a/srcxray.py +++ b/srcxray.py @@ -864,14 +864,16 @@ def add_rank(g): def usage(): + print("Usage:\n") for c in ["referers_tree", "call_tree", "referers_dep", "call_dep"]: print(me, c, "") - print("Try this:") + print("\nTry this:\n") print("cd linux/init") print(me, "unittest") print(me, "referers_tree nfs_root_data") print(me, "call_tree start_kernel") - print(me, "Emergency termination: ^Z, kill %1") + print("Emergency termination: ^Z, kill %1") + print() class _unittest_autotest(unittest.TestCase): From 37b2361fa5ceafe6e63b8e1a84c6ffa02b837669 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 11 Aug 2018 17:52:02 +0300 Subject: [PATCH 62/66] move , --- srcxray.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/srcxray.py b/srcxray.py index 4c2fe71..1dc050a 100755 --- a/srcxray.py +++ b/srcxray.py @@ -59,7 +59,7 @@ 'retry LOOKUP_REVAL retry_estale user_path_at lookup_flags old ' 'current_user_ns spin_lock_irq spin_unlock_irq prepare_creds ' 'tasklist_lock commit_creds read_lock read_unlock SIGKILL SIGSTOP abort_creds fd_install ' - 'real_mount FMODE_WRITE tv_nsec putname ' + 'real_mount FMODE_WRITE tv_nsec putname ,' ).split() @@ -639,8 +639,6 @@ def write_dot(g, dot): g.remove_nodes_from(black_list) ranks = collections.defaultdict(list) for n in g.nodes(): - if n == ',': - continue r = rank(g, n) if r: ranks[r].append(n) @@ -649,8 +647,7 @@ def write_dot(g, dot): # dot.write((n if n != 'node' else '"node"') + ';\n') # dot.write((n if n != 'node' else '"node"') + ' -> { ') dot.write('"%s" -> { ' % (n)) - # dot.write(' '.join([str(a) if a != 'node' else '"node"' for a in g.successors(n) if str(a) != ','])) - dot.write(' '.join(['"%s"' % (str(a)) for a in g.successors(n) if str(a) != ','])) + dot.write(' '.join(['"%s"' % (str(a)) for a in g.successors(n)])) if r and scaled: dot.write(' } [penwidth=%d label=%d];\n' % (100/r, r)) else: From 7423670b03dc7da7bf63e896c33de269a325f08f Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sat, 11 Aug 2018 17:52:46 +0300 Subject: [PATCH 63/66] cleanup --- srcxray.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/srcxray.py b/srcxray.py index 1dc050a..13ac375 100755 --- a/srcxray.py +++ b/srcxray.py @@ -644,8 +644,6 @@ def write_dot(g, dot): ranks[r].append(n) if not g.out_degree(n): continue - # dot.write((n if n != 'node' else '"node"') + ';\n') - # dot.write((n if n != 'node' else '"node"') + ' -> { ') dot.write('"%s" -> { ' % (n)) dot.write(' '.join(['"%s"' % (str(a)) for a in g.successors(n)])) if r and scaled: From 154419de8180c8ac4cbc13d1441efe657834fb0c Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 12 Aug 2018 23:27:49 +0300 Subject: [PATCH 64/66] unittest +write_dot +read_dot2 --- srcxray.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/srcxray.py b/srcxray.py index 13ac375..1a1f73d 100755 --- a/srcxray.py +++ b/srcxray.py @@ -873,6 +873,9 @@ def usage(): class _unittest_autotest(unittest.TestCase): def test_1(self): + write_dot(nx.DiGraph([(1, 2), (2, 3), (2, 4)]), 'test.dot') + g = read_dot2("test.dot") + self.assertEqual(list(g.successors("2")), ["3", "4"]) self.assertTrue(os.path.isdir('include/linux/')) os.chdir('init') self.assertEqual('\t\t\t\t\tprepare_namespace ^', popen('srcxray.py referers_tree nfs_root_data')[-1]) From 2d0e22045262998c5fd4cb67212a9a24800f3864 Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 12 Aug 2018 23:28:19 +0300 Subject: [PATCH 65/66] stats: +couples --- srcxray.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/srcxray.py b/srcxray.py index 1a1f73d..1f47be6 100755 --- a/srcxray.py +++ b/srcxray.py @@ -787,6 +787,7 @@ def stats(a): leaves = set() roots = set() stat.edge_nodes = 0 + stat.couples = 0 for n in dg: id = dg.in_degree(n) od = dg.out_degree(n) @@ -800,6 +801,8 @@ def stats(a): om[n] = od else: leaves.add(n) + if od == 1 and dg.in_degree(list(dg.successors(n))[0]) == 1: + stat.couples += 1 stat.max_in_degree = max(dict(dg.in_degree).values()) stat.max_out_degree = max(dict(dg.out_degree).values()) stat.leaves = len(leaves) From eb05c480edb781cf8325f2c08a1e0ba88643792d Mon Sep 17 00:00:00 2001 From: Constantine Shulyupin Date: Sun, 12 Aug 2018 23:28:53 +0300 Subject: [PATCH 66/66] *write_dot --- srcxray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/srcxray.py b/srcxray.py index 1f47be6..76728eb 100755 --- a/srcxray.py +++ b/srcxray.py @@ -658,7 +658,8 @@ def write_dot(g, dot): if scaled and len(ranks): prop.fontsize = 500 + 10000 / (len(ranks[rank(g, n)]) + 1) # prop.label = n + ' ' + str(rank(g,n)) - dot.write('"%s" [%s]\n' % (n, ','.join(['%s="%s"' % (a, str(prop[a])) for a in prop]))) + if prop: + dot.write('"%s" [%s]\n' % (n, ','.join(['%s="%s"' % (a, str(prop[a])) for a in prop]))) # else: # dot.write('"%s"\n'%(n)) dot.write('}\n')