KiCad PCB EDA Suite
github_getliblist.cpp
Go to the documentation of this file.
1 /*
2  * This program source code file is part of KiCad, a free EDA CAD application.
3  *
4  * Copyright (C) 2015 Jean-Pierre Charras jp.charras at wanadoo.fr
5  * Copyright (C) 2013 SoftPLC Corporation, Dick Hollenbeck <dick@softplc.com>
6  * Copyright (C) 2015 KiCad Developers, see CHANGELOG.TXT for contributors.
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License
10  * as published by the Free Software Foundation; either version 2
11  * of the License, or (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, you may find one here:
20  * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
21  * or you may search the http://www.gnu.org website for the version 2 license,
22  * or you may write to the Free Software Foundation, Inc.,
23  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
24  */
25 
26 
27 /*
28  * While creating a wizard to edit the fp lib tables, and mainly the web viewer
29  * which can read the list of pretty library on a github repos, I was told there is
30  * this URL to retrieve info from any particular repo:
31  *
32  * https://api.github.com/orgs/KiCad/repos
33  * or
34  * https://api.github.com/users/KiCad/repos
35  *
36  * This gets just information on the repo in JSON format.
37  *
38  * I used avhttp, already used in the pcbnew Github plugin to download
39  * the json file.
40  *
41  * JP Charras.
42  */
43 
44 #include <kicad_curl/kicad_curl_easy.h> // Include before any wx file
45 #include <wx/uri.h>
46 
47 #include <github_getliblist.h>
48 #include <macros.h>
49 #include <common.h>
50 #include <html_link_parser.h>
51 
52 
53 GITHUB_GETLIBLIST::GITHUB_GETLIBLIST( const wxString& aRepoURL )
54 {
55  m_repoURL = aRepoURL;
56  m_libs_ext = wxT( ".pretty" );
57  strcpy( m_option_string, "application/json" );
58 }
59 
60 
61 bool GITHUB_GETLIBLIST::Get3DshapesLibsList( wxArrayString* aList,
62  bool (*aFilter)( const wxString& aData ) )
63 {
64  std::string fullURLCommand;
65 
66  strcpy( m_option_string, "text/html" );
67 
68  wxString repoURL = m_repoURL;
69 
70  wxString errorMsg;
71 
72  fullURLCommand = repoURL.utf8_str();
73  bool success = remoteGetJSON( fullURLCommand, &errorMsg );
74 
75  if( !success )
76  {
77  wxMessageBox( errorMsg );
78  return false;
79  }
80 
81  if( aFilter && aList )
82  {
83  //Convert m_image (std::string) to a wxString for HTML_LINK_PARSER
84  wxString buffer( GetBuffer() );
85 
86  HTML_LINK_PARSER html_parser( buffer, *aList );
87  html_parser.ParseLinks( aFilter );
88  }
89 
90  return true;
91 }
92 
93 
94 bool GITHUB_GETLIBLIST::GetFootprintLibraryList( wxArrayString& aList )
95 {
96  std::string fullURLCommand;
97  int page = 1;
98  int itemCountMax = 99; // Do not use a valu > 100, it does not work
99 
100  strcpy( m_option_string, "application/json" );
101 
102  // Github max items returned is 100 per page
103 
104  if( !repoURL2listURL( m_repoURL, &fullURLCommand, itemCountMax, page ) )
105  {
106  wxString msg = wxString::Format( _( "malformed URL:\n'%s'" ), GetChars( m_repoURL ) );
107  wxMessageBox( msg );
108  return false;
109  }
110 
111  // The URL lib names are relative to the server name.
112  // so add the server name to them.
113  wxURI repo( m_repoURL );
114  wxString urlPrefix = repo.GetScheme() + wxT( "://" ) + repo.GetServer() + wxT( "/" );
115 
116  wxString errorMsg;
117  const char sep = ','; // Separator fields, in json returned file
118  wxString tmp;
119  int items_count_per_page = 0;
120  std::string& json_image = GetBuffer();
121 
122  while( 1 )
123  {
124  bool success = remoteGetJSON( fullURLCommand, &errorMsg );
125 
126  if( !success )
127  {
128  wxMessageBox( errorMsg );
129  return false;
130  }
131 
132 
133  for( unsigned ii = 0; ii < json_image.size(); ii++ )
134  {
135  if( json_image[ii] == sep || ii == json_image.size() - 1 )
136  {
137  if( tmp.StartsWith( wxT( "\"full_name\"" ) ) )
138  {
139  #define QUOTE '\"'
140  // Remove useless quotes:
141  if( tmp[tmp.Length() - 1] == QUOTE )
142  tmp.RemoveLast();
143 
144  if( tmp.EndsWith( m_libs_ext ) )
145  {
146  aList.Add( tmp.AfterLast( ':' ) );
147  int idx = aList.GetCount() - 1;
148 
149  if( aList[idx][0] == QUOTE )
150  aList[idx].Remove( 0, 1 );
151 
152  aList[idx].Prepend( urlPrefix );
153  }
154 
155  items_count_per_page++;
156  }
157 
158  tmp.Clear();
159  }
160  else
161  tmp << json_image[ii];
162  }
163 
164  if( items_count_per_page >= itemCountMax )
165  {
166  page++;
167  repoURL2listURL( m_repoURL, &fullURLCommand, itemCountMax, page );
168  items_count_per_page = 0;
169  ClearBuffer();
170  }
171  else
172  break;
173  }
174 
175  aList.Sort();
176  return true;
177 }
178 
179 
180 bool GITHUB_GETLIBLIST::repoURL2listURL( const wxString& aRepoURL,
181  std::string* aFullURLCommand,
182  int aItemCountMax, int aPage )
183 {
184  // aListURL is e.g. "https://api.github.com/orgs/KiCad/repos"
185  // or "https://api.github.com/users/KiCad/repos"
186  // aRepoURL is e.g. "https://github.com/KiCad"
187  // Github has a default pagination set to 30 items.
188  // but allows up to 100 items max if we add the "?per_page=100" option
189 
190  wxURI repo( aRepoURL );
191 
192  if( repo.HasServer() && repo.HasPath() )
193  {
194  // goal: "https://api.github.com/users/KiCad"
195  // or "https://api.github.com/orgs/KiCad"
196  // "https://api.github.com/users/..." works both for orgs and users
197  // if we just retrieve the .pretty list
198  wxString target_url( wxT( "https://api.github.com/users" ) );
199  target_url += repo.GetPath();
200  target_url += wxT( "/repos" );
201 
202  // Github has a default pagination set to 30 items.
203  // but allows up to 100 items max. Use this limit
204  target_url += wxString::Format( "?per_page=%d&page=%d", aItemCountMax, aPage );
205 
206  *aFullURLCommand = target_url.utf8_str();
207  return true;
208  }
209 
210  return false;
211 }
212 
213 
214 bool GITHUB_GETLIBLIST::remoteGetJSON( const std::string& aFullURLCommand, wxString* aMsgError )
215 {
216  KICAD_CURL_EASY kcurl;
217 
218  wxLogDebug( wxT( "Attempting to download: " ) + aFullURLCommand );
219 
220  kcurl.SetURL( aFullURLCommand );
221  kcurl.SetUserAgent( "http://kicad-pcb.org" );
222  kcurl.SetHeader( "Accept", m_option_string );
223  kcurl.SetFollowRedirects( true );
224 
225  try
226  {
227  kcurl.Perform();
228  m_image = kcurl.GetBuffer();
229  return true;
230  }
231  catch( const IO_ERROR& ioe )
232  {
233  if( aMsgError )
234  {
235  UTF8 fmt( _( "Error fetching JSON data from URL '%s'.\nReason: '%s'" ) );
236 
237  std::string msg = StrPrintf( fmt.c_str(),
238  aFullURLCommand.c_str(),
239  TO_UTF8( ioe.What() ) );
240 
241  *aMsgError = FROM_UTF8( msg.c_str() );
242  }
243  return false;
244  }
245 }
Class UTF8 is an 8 bit std::string that is assuredly encoded in UTF8, and supplies special conversion...
Definition: utf8.h:53
bool SetUserAgent(const std::string &aAgent)
Function SetUserAgent sets the request user agent.
void Perform()
Function perform equivalent to curl_easy_perform.
bool SetFollowRedirects(bool aFollow)
Function SetFollowRedirects enables the following of HTTP(s) and other redirects, by default curl doe...
GITHUB_GETLIBLIST(const wxString &aRepoURL)
static wxString FROM_UTF8(const char *cstring)
function FROM_UTF8 converts a UTF8 encoded C string to a wxString for all wxWidgets build modes...
Definition: macros.h:53
Class KICAD_CURL_EASY wrapper interface around the curl_easy API.
char m_option_string[64]
option for transfert type, like "application/json"
bool GetFootprintLibraryList(wxArrayString &aList)
Fills aList by the name of footprint libraries found on the github repo.
This file contains miscellaneous commonly used macros and functions.
int StrPrintf(std::string *result, const char *format,...)
Function StrPrintf is like sprintf() but the output is appended to a std::string instead of to a char...
Definition: richio.cpp:75
#define TO_UTF8(wxstring)
Macro TO_UTF8 converts a wxString to a UTF8 encoded C string for all wxWidgets build modes...
Definition: macros.h:47
const std::string & GetBuffer()
Function GetBuffer returns a const reference to the recevied data buffer.
bool Get3DshapesLibsList(wxArrayString *aList, bool(*aFilter)(const wxString &aData))
Fills aList by the URL of libraries found on the github repo.
std::string m_image
image of the downloaded data in its entirety.
bool SetURL(const std::string &aURL)
Function SetURL sets the request URL.
bool repoURL2listURL(const wxString &aRepoURL, std::string *aFullURLCommand, int aItemCountMax, int aPage=1)
Function repoURL2listURL translates a repo URL to the URL name which gives the state of repos URL as ...
#define QUOTE
virtual const wxString What() const
A composite of Problem() and Where()
Definition: exceptions.cpp:33
static const wxChar * GetChars(const wxString &s)
Function GetChars returns a wxChar* to the actual wxChar* data within a wxString, and is helpful for ...
Definition: macros.h:92
void Format(OUTPUTFORMATTER *out, int aNestLevel, int aCtl, CPTREE &aTree)
Function Format outputs a PTREE into s-expression format via an OUTPUTFORMATTER derivative.
Definition: ptree.cpp:205
bool remoteGetJSON(const std::string &aFullURLCommand, wxString *aMsgError)
Function remoteGetJSON Download a json text from a github repo.
wxString m_repoURL
the URL of the Github repo
wxString m_libs_ext
the extension of the name of libraries (default = .pretty)
The common library.
void ClearBuffer()
Clear the buffer which stores all the downloaded raw data.
void SetHeader(const std::string &aName, const std::string &aValue)
Function SetHeader sets an arbitrary header for the HTTP(s) request.
Struct IO_ERROR is a class used to hold an error message and may be used when throwing exceptions con...
Definition: ki_exception.h:47
std::string & GetBuffer()